refactor: apply modernize-use-override across codebase (#442)
- Remove redundant `virtual` keyword from methods already marked `override` - Replace `virtual ~Derived()` with `~Derived() override` for derived class destructors - Add missing `override` on methods that override base class virtuals
This commit is contained in:
@@ -27,7 +27,7 @@ template <size_t BATCH_SIZE>
|
||||
class FlatBuilder : public IndexBuilder {
|
||||
public:
|
||||
//! Destructor
|
||||
virtual ~FlatBuilder(void) {}
|
||||
~FlatBuilder(void) override {}
|
||||
|
||||
//! Initialize the builder
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
@@ -27,7 +27,7 @@ template <size_t BATCH_SIZE>
|
||||
class FlatSearcher : public IndexSearcher {
|
||||
public:
|
||||
//! Destructor
|
||||
virtual ~FlatSearcher(void) = default;
|
||||
~FlatSearcher(void) override = default;
|
||||
|
||||
//! Initialize Searcher
|
||||
int init(const ailego::Params &index_params) override {
|
||||
|
||||
@@ -35,7 +35,7 @@ class FlatSearcherContext : public IndexSearcher::Context {
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~FlatSearcherContext(void) {}
|
||||
~FlatSearcherContext(void) override {}
|
||||
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t topk) override {
|
||||
@@ -58,13 +58,12 @@ class FlatSearcherContext : public IndexSearcher::Context {
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
const IndexGroupDocumentList &group_result(size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ class FlatStreamer : public IndexStreamer {
|
||||
using ContextPointer = IndexStreamer::Context::UPointer;
|
||||
|
||||
FlatStreamer(void);
|
||||
virtual ~FlatStreamer(void);
|
||||
~FlatStreamer(void) override;
|
||||
|
||||
FlatStreamer(const FlatStreamer &streamer) = delete;
|
||||
FlatStreamer &operator=(const FlatStreamer &streamer) = delete;
|
||||
@@ -126,12 +126,12 @@ class FlatStreamer : public IndexStreamer {
|
||||
return entity_;
|
||||
}
|
||||
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return this->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return this->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ class FlatStreamerContext : public IndexStreamer::Context {
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
virtual ~FlatStreamerContext(void) = default;
|
||||
~FlatStreamerContext(void) override = default;
|
||||
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t topk) override {
|
||||
@@ -59,12 +59,11 @@ class FlatStreamerContext : public IndexStreamer::Context {
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
const IndexGroupDocumentList &group_result(size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ class FlatSparseContext : public IndexContext {
|
||||
: searcher_owner_(searcher_ptr), context_type_(kSearcherContext) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~FlatSparseContext(void) = default;
|
||||
~FlatSparseContext(void) override = default;
|
||||
|
||||
//! Set topk of search result
|
||||
void set_topk(uint32_t topk) override {
|
||||
|
||||
@@ -100,33 +100,33 @@ class FlatSparseIndexProvider : public IndexSparseProvider {
|
||||
}
|
||||
|
||||
//! Retrieve sparse count
|
||||
virtual uint32_t sparse_count() const override {
|
||||
uint32_t sparse_count() const override {
|
||||
return sparse_count_;
|
||||
}
|
||||
|
||||
//! Retrieve sparse indices
|
||||
virtual const uint32_t *sparse_indices() const override {
|
||||
const uint32_t *sparse_indices() const override {
|
||||
return reinterpret_cast<const uint32_t *>(sparse_indices_buffer_.data());
|
||||
}
|
||||
|
||||
//! Retrieve sparse data
|
||||
virtual const void *sparse_data() const override {
|
||||
const void *sparse_data() const override {
|
||||
return reinterpret_cast<const void *>(sparse_data_buffer_.data());
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt() && valid_;
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
// std::cout << "iter key=" << cur_id_ << std::endl;
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
|
||||
if (cur_id_ < entity_->doc_cnt()) {
|
||||
|
||||
@@ -28,7 +28,7 @@ class FlatSparseSearcher : public IndexSearcher {
|
||||
|
||||
public:
|
||||
FlatSparseSearcher(void);
|
||||
virtual ~FlatSparseSearcher(void);
|
||||
~FlatSparseSearcher(void) override;
|
||||
|
||||
FlatSparseSearcher(const FlatSparseSearcher &) = delete;
|
||||
FlatSparseSearcher &operator=(const FlatSparseSearcher &) = delete;
|
||||
|
||||
@@ -31,7 +31,7 @@ class FlatSparseStreamer : public IndexStreamer {
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
FlatSparseStreamer(void);
|
||||
virtual ~FlatSparseStreamer(void);
|
||||
~FlatSparseStreamer(void) override;
|
||||
|
||||
FlatSparseStreamer(const FlatSparseStreamer &streamer) = delete;
|
||||
FlatSparseStreamer &operator=(const FlatSparseStreamer &streamer) = delete;
|
||||
|
||||
@@ -43,58 +43,57 @@ class HnswContext : public IndexContext {
|
||||
const HnswEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~HnswContext();
|
||||
~HnswContext() override;
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
const IndexGroupDocumentList &group_result(size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
bool debug_mode(void) const override {
|
||||
return this->debugging();
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(
|
||||
buf, sizeof(buf),
|
||||
@@ -105,7 +104,7 @@ class HnswContext : public IndexContext {
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
int update(const ailego::Params ¶ms) override;
|
||||
|
||||
public:
|
||||
//! Init context
|
||||
|
||||
@@ -82,22 +82,22 @@ class HnswIndexProvider : public IndexProvider {
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
// cur_id_ += 1;
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ class HnswStreamer : public IndexStreamer {
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
HnswStreamer(void);
|
||||
virtual ~HnswStreamer(void);
|
||||
~HnswStreamer(void) override;
|
||||
|
||||
HnswStreamer(const HnswStreamer &streamer) = delete;
|
||||
HnswStreamer &operator=(const HnswStreamer &streamer) = delete;
|
||||
@@ -44,102 +44,99 @@ class HnswStreamer : public IndexStreamer {
|
||||
|
||||
protected:
|
||||
//! Initialize Streamer
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) override;
|
||||
int init(const IndexMeta &imeta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Streamer
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Create a context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Add a vector into index
|
||||
virtual int add_impl(uint64_t pkey, const void *query,
|
||||
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Add a vector with id into index
|
||||
int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Add a vector with id into index
|
||||
virtual int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Fetch vector by key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Fetch vector by id
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector(id, block);
|
||||
}
|
||||
|
||||
//! Open index from file path
|
||||
virtual int open(IndexStorage::Pointer stg) override;
|
||||
int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
//! Close file
|
||||
virtual int close(void) override;
|
||||
int close(void) override;
|
||||
|
||||
//! flush file
|
||||
virtual int flush(uint64_t checkpoint) override;
|
||||
int flush(uint64_t checkpoint) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
inline int check_params(const void *query,
|
||||
|
||||
@@ -43,56 +43,54 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
public:
|
||||
//! Cleanup
|
||||
//! return 0 on success, or errCode in failure
|
||||
virtual int cleanup() override;
|
||||
int cleanup() override;
|
||||
|
||||
//! Make a copy of streamer entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswEntity::Pointer clone() const override;
|
||||
const HnswEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
const void *get_vector(node_id_t id) const override;
|
||||
|
||||
//! Get vectors feature data by local ids
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
virtual int get_vector(
|
||||
int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
//! Note: the neighbors cannot be modified, using the following
|
||||
//! method to get WritableNeighbors if want to
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
|
||||
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
|
||||
//! Add vector and id to hnsw entity
|
||||
virtual int add_vector_with_id(level_t level, node_id_t id,
|
||||
const void *vec) override;
|
||||
int add_vector_with_id(level_t level, node_id_t id, const void *vec) override;
|
||||
|
||||
virtual int update_neighbors(
|
||||
int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
|
||||
//! Append neighbor_id to node id neighbors on level
|
||||
//! Notice: the caller must be ensure the neighbors not full
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
|
||||
//! Get the storage mode of this entity
|
||||
virtual HnswStorageMode storage_mode() const {
|
||||
@@ -112,13 +110,13 @@ class HnswStreamerEntity : public HnswEntity {
|
||||
~HnswStreamerEntity();
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_by_key(key_t key) const override {
|
||||
const void *get_vector_by_key(key_t key) const override {
|
||||
auto id = get_id(key);
|
||||
return id == kInvalidNodeId ? nullptr : get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(
|
||||
const key_t key, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_key(const key_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
auto id = get_id(key);
|
||||
if (id != kInvalidNodeId) {
|
||||
return get_vector(id, block);
|
||||
|
||||
@@ -44,58 +44,57 @@ class HnswRabitqContext : public IndexContext {
|
||||
const HnswRabitqEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~HnswRabitqContext();
|
||||
~HnswRabitqContext() override;
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
const IndexGroupDocumentList &group_result(size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
bool debug_mode(void) const override {
|
||||
return this->debugging();
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(
|
||||
buf, sizeof(buf),
|
||||
@@ -106,7 +105,7 @@ class HnswRabitqContext : public IndexContext {
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
int update(const ailego::Params ¶ms) override;
|
||||
|
||||
public:
|
||||
//! Init context
|
||||
|
||||
@@ -83,22 +83,22 @@ class HnswRabitqIndexProvider : public IndexProvider {
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(cur_id_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
// cur_id_ += 1;
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ class HnswRabitqStreamer : public IndexStreamer {
|
||||
HnswRabitqStreamer();
|
||||
explicit HnswRabitqStreamer(IndexProvider::Pointer provider,
|
||||
RabitqReformer::Pointer reformer = nullptr);
|
||||
virtual ~HnswRabitqStreamer(void);
|
||||
~HnswRabitqStreamer(void) override;
|
||||
|
||||
HnswRabitqStreamer(const HnswRabitqStreamer &streamer) = delete;
|
||||
HnswRabitqStreamer &operator=(const HnswRabitqStreamer &streamer) = delete;
|
||||
@@ -48,102 +48,99 @@ class HnswRabitqStreamer : public IndexStreamer {
|
||||
|
||||
protected:
|
||||
//! Initialize Streamer
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) override;
|
||||
int init(const IndexMeta &imeta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup Streamer
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Create a context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Add a vector into index
|
||||
virtual int add_impl(uint64_t pkey, const void *query,
|
||||
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Add a vector with id into index
|
||||
int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
//! Add a vector with id into index
|
||||
virtual int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
//! Similarity search
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
//! Linear search by primary keys
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
//! Fetch vector by key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_.get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
//! Fetch vector by id
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_.get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_.get_vector(id, block);
|
||||
}
|
||||
|
||||
//! Open index from file path
|
||||
virtual int open(IndexStorage::Pointer stg) override;
|
||||
int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
//! Close file
|
||||
virtual int close(void) override;
|
||||
int close(void) override;
|
||||
|
||||
//! flush file
|
||||
virtual int flush(uint64_t checkpoint) override;
|
||||
int flush(uint64_t checkpoint) override;
|
||||
|
||||
//! Dump index into storage
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
inline int check_params(const void *query,
|
||||
|
||||
@@ -34,57 +34,54 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
|
||||
public:
|
||||
//! Cleanup
|
||||
//! return 0 on success, or errCode in failure
|
||||
virtual int cleanup() override;
|
||||
int cleanup() override;
|
||||
|
||||
//! Make a copy of streamer entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswRabitqEntity::Pointer clone() const override;
|
||||
const HnswRabitqEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
const void *get_vector(node_id_t id) const override;
|
||||
|
||||
//! Get vectors feature data by local ids
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
virtual int get_vector(
|
||||
int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
//! Note: the neighbors cannot be modified, using the following
|
||||
//! method to get WritableNeighbors if want to
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
|
||||
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
int add_vector(level_t level, key_t key, const void *vec,
|
||||
node_id_t *id) override;
|
||||
|
||||
//! Add vector and id to hnsw entity
|
||||
virtual int add_vector_with_id(level_t level, node_id_t id,
|
||||
const void *vec) override;
|
||||
int add_vector_with_id(level_t level, node_id_t id, const void *vec) override;
|
||||
|
||||
virtual int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>> &neighbors)
|
||||
override;
|
||||
int update_neighbors(level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, ResultRecord>>
|
||||
&neighbors) override;
|
||||
|
||||
//! Append neighbor_id to node id neighbors on level
|
||||
//! Notice: the caller must be ensure the neighbors not full
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
|
||||
void set_use_key_info_map(bool use_id_map) {
|
||||
use_key_info_map_ = use_id_map;
|
||||
@@ -99,13 +96,13 @@ class HnswRabitqStreamerEntity : public HnswRabitqEntity {
|
||||
~HnswRabitqStreamerEntity();
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_by_key(key_t key) const override {
|
||||
const void *get_vector_by_key(key_t key) const override {
|
||||
auto id = get_id(key);
|
||||
return id == kInvalidNodeId ? nullptr : get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(
|
||||
const key_t key, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_key(const key_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
auto id = get_id(key);
|
||||
if (id != kInvalidNodeId) {
|
||||
return get_vector(id, block);
|
||||
|
||||
@@ -22,23 +22,23 @@ namespace core {
|
||||
class HnswSparseBuilderEntity : public HnswSparseEntity {
|
||||
public:
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t level, key_t key, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices, const void *sparse_vec,
|
||||
node_id_t *id) override;
|
||||
int add_vector(level_t level, key_t key, const uint32_t sparse_count,
|
||||
const uint32_t *sparse_indices, const void *sparse_vec,
|
||||
node_id_t *id) override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_meta(node_id_t id) const override;
|
||||
const void *get_vector_meta(node_id_t id) const override;
|
||||
|
||||
virtual int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Batch get vectors feature data by keys
|
||||
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
virtual int get_vector_metas(
|
||||
int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector_metas(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
|
||||
|
||||
@@ -57,43 +57,41 @@ class HnswSparseBuilderEntity : public HnswSparseEntity {
|
||||
}
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
|
||||
|
||||
//! Replace node id in level's neighbors
|
||||
virtual int update_neighbors(
|
||||
int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
|
||||
//! add a neighbor to id in graph level
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Get vector sparse feature data by chunk index and offset
|
||||
virtual const void *get_sparse_data(uint64_t offset,
|
||||
uint32_t len) const override;
|
||||
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
|
||||
//! Get sparse data from id
|
||||
virtual const void *get_sparse_data(node_id_t id) const override;
|
||||
const void *get_sparse_data(node_id_t id) const override;
|
||||
|
||||
virtual int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
virtual int get_sparse_data(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Get sparse data from vector
|
||||
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
const void *vec) const override;
|
||||
|
||||
virtual int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
|
||||
//! Dump the hnsw graph to dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Cleanup the entity
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
|
||||
@@ -38,58 +38,57 @@ class HnswSparseContext : public IndexContext {
|
||||
const HnswSparseEntity::Pointer &entity);
|
||||
|
||||
//! Destructor
|
||||
virtual ~HnswSparseContext();
|
||||
~HnswSparseContext() override;
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve result object for output
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(void) const override {
|
||||
const IndexGroupDocumentList &group_result(void) const override {
|
||||
return group_results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search group result with index
|
||||
virtual const IndexGroupDocumentList &group_result(
|
||||
size_t idx) const override {
|
||||
const IndexGroupDocumentList &group_result(size_t idx) const override {
|
||||
return group_results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
//! Set mode of debug
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
|
||||
//! Retrieve mode of debug
|
||||
virtual bool debug_mode(void) const override {
|
||||
bool debug_mode(void) const override {
|
||||
return this->debugging();
|
||||
}
|
||||
|
||||
//! Retrieve string of debug
|
||||
virtual std::string debug_string(void) const override {
|
||||
std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(
|
||||
buf, sizeof(buf),
|
||||
@@ -100,7 +99,7 @@ class HnswSparseContext : public IndexContext {
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
int update(const ailego::Params ¶ms) override;
|
||||
|
||||
public:
|
||||
//! Init context
|
||||
@@ -313,7 +312,7 @@ class HnswSparseContext : public IndexContext {
|
||||
force_padding_topk_ = v;
|
||||
}
|
||||
|
||||
virtual void set_bruteforce_threshold(uint32_t v) override {
|
||||
void set_bruteforce_threshold(uint32_t v) override {
|
||||
bruteforce_threshold_ = v;
|
||||
}
|
||||
|
||||
@@ -321,11 +320,11 @@ class HnswSparseContext : public IndexContext {
|
||||
return bruteforce_threshold_;
|
||||
}
|
||||
|
||||
virtual void set_fetch_vector(bool v) override {
|
||||
void set_fetch_vector(bool v) override {
|
||||
fetch_vector_ = v;
|
||||
}
|
||||
|
||||
virtual bool fetch_vector() const override {
|
||||
bool fetch_vector() const override {
|
||||
return fetch_vector_;
|
||||
}
|
||||
|
||||
|
||||
@@ -81,32 +81,32 @@ class HnswSparseIndexProvider : public IndexSparseProvider {
|
||||
}
|
||||
|
||||
//! Retrieve sparse count
|
||||
virtual uint32_t sparse_count() const override {
|
||||
uint32_t sparse_count() const override {
|
||||
return sparse_count_;
|
||||
}
|
||||
|
||||
//! Retrieve sparse indices
|
||||
virtual const uint32_t *sparse_indices() const override {
|
||||
const uint32_t *sparse_indices() const override {
|
||||
return reinterpret_cast<const uint32_t *>(sparse_indices_buffer_.data());
|
||||
}
|
||||
|
||||
//! Retrieve sparse data
|
||||
virtual const void *sparse_data() const override {
|
||||
const void *sparse_data() const override {
|
||||
return reinterpret_cast<const void *>(sparse_data_buffer_.data());
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return cur_id_ < entity_->doc_cnt() && valid_;
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(cur_id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
cur_id_ = get_next_valid_id(cur_id_ + 1);
|
||||
|
||||
if (cur_id_ < entity_->doc_cnt()) {
|
||||
|
||||
@@ -26,7 +26,7 @@ class HnswSparseSearcher : public IndexSearcher {
|
||||
|
||||
public:
|
||||
HnswSparseSearcher(void);
|
||||
virtual ~HnswSparseSearcher(void);
|
||||
~HnswSparseSearcher(void) override;
|
||||
|
||||
HnswSparseSearcher(const HnswSparseSearcher &) = delete;
|
||||
HnswSparseSearcher &operator=(const HnswSparseSearcher &) = delete;
|
||||
|
||||
@@ -46,64 +46,61 @@ class HnswSparseSearcherEntity : public HnswSparseEntity {
|
||||
|
||||
//! Make a copy of searcher entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswSparseEntity::Pointer clone() const override;
|
||||
const HnswSparseEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector local id by key
|
||||
node_id_t get_id(key_t key) const;
|
||||
|
||||
//! Get sparse vector feature data by key
|
||||
virtual int get_sparse_vector_by_key(
|
||||
int get_sparse_vector_by_key(
|
||||
key_t key, uint32_t *sparse_count, std::string *sparse_indices_buffer,
|
||||
std::string *sparse_values_buffer) const override;
|
||||
|
||||
//! Get vector feature data by id
|
||||
virtual const void *get_vector_meta(node_id_t id) const override;
|
||||
const void *get_vector_meta(node_id_t id) const override;
|
||||
|
||||
virtual int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Get vector feature data by id
|
||||
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
|
||||
virtual int get_vector_metas(
|
||||
int get_vector_metas(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
|
||||
|
||||
//! Get vector sparse feature data by chunk index and offset
|
||||
virtual const void *get_sparse_data(uint64_t offset,
|
||||
uint32_t len) const override;
|
||||
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
|
||||
|
||||
//! Get sparse data from id
|
||||
virtual const void *get_sparse_data(node_id_t id) const override;
|
||||
const void *get_sparse_data(node_id_t id) const override;
|
||||
|
||||
virtual int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
virtual int get_sparse_data(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Get sparse data from vector
|
||||
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
const void *vec) const override;
|
||||
|
||||
virtual int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
|
||||
|
||||
virtual int load(const IndexStorage::Pointer &container,
|
||||
bool check_crc) override;
|
||||
int load(const IndexStorage::Pointer &container, bool check_crc) override;
|
||||
|
||||
int load_segments(bool check_crc);
|
||||
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
public:
|
||||
bool is_loaded() const {
|
||||
|
||||
@@ -26,7 +26,7 @@ class HnswSparseStreamer : public IndexStreamer {
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
HnswSparseStreamer(void);
|
||||
virtual ~HnswSparseStreamer(void);
|
||||
~HnswSparseStreamer(void) override;
|
||||
|
||||
HnswSparseStreamer(const HnswSparseStreamer &streamer) = delete;
|
||||
HnswSparseStreamer &operator=(const HnswSparseStreamer &streamer) = delete;
|
||||
|
||||
@@ -32,75 +32,72 @@ class HnswSparseStreamerEntity : public HnswSparseEntity {
|
||||
public:
|
||||
//! Cleanup
|
||||
//! return 0 on success, or errCode in failure
|
||||
virtual int cleanup() override;
|
||||
int cleanup() override;
|
||||
|
||||
//! Make a copy of streamer entity, to support thread-safe operation.
|
||||
//! The segment in container cannot be read concurrenly
|
||||
virtual const HnswSparseEntity::Pointer clone() const override;
|
||||
const HnswSparseEntity::Pointer clone() const override;
|
||||
|
||||
//! Get primary key of the node id
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
|
||||
//! Get vector feature data by key
|
||||
virtual const void *get_vector_meta(node_id_t id) const override;
|
||||
const void *get_vector_meta(node_id_t id) const override;
|
||||
|
||||
virtual int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector_meta(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Get vectors feature data by local ids
|
||||
virtual int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
virtual int get_vector_metas(
|
||||
int get_vector_metas(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector_metas(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &block_vecs) const override;
|
||||
|
||||
//! Get vector sparse feature data by chunk index and offset
|
||||
virtual const void *get_sparse_data(uint64_t offset,
|
||||
uint32_t len) const override;
|
||||
const void *get_sparse_data(uint64_t offset, uint32_t len) const override;
|
||||
|
||||
//! Get sparse data from id
|
||||
virtual const void *get_sparse_data(node_id_t id) const override;
|
||||
const void *get_sparse_data(node_id_t id) const override;
|
||||
|
||||
virtual int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
virtual int get_sparse_data(node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(uint64_t offset, uint32_t len,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_sparse_data(node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
|
||||
//! Get sparse data from vector
|
||||
virtual std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
std::pair<const void *, uint32_t> get_sparse_data_from_vector(
|
||||
const void *vec) const override;
|
||||
virtual int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
int get_sparse_data_from_vector(const void *vec,
|
||||
IndexStorage::MemoryBlock &block,
|
||||
int &sparse_length) const override;
|
||||
|
||||
//! Get sparse vector feature data by key
|
||||
virtual int get_sparse_vector_by_key(
|
||||
int get_sparse_vector_by_key(
|
||||
key_t key, uint32_t *sparse_count, std::string *sparse_indices_buffer,
|
||||
std::string *sparse_values_buffer) const override;
|
||||
|
||||
//! Get sparse vector feature data by id
|
||||
virtual int get_sparse_vector_by_id(
|
||||
node_id_t id, uint32_t *sparse_count, std::string *sparse_indices_buffer,
|
||||
std::string *sparse_values_buffer) const override;
|
||||
int get_sparse_vector_by_id(node_id_t id, uint32_t *sparse_count,
|
||||
std::string *sparse_indices_buffer,
|
||||
std::string *sparse_values_buffer) const override;
|
||||
|
||||
//! Get the node id's neighbors on graph level
|
||||
//! Note: the neighbors cannot be modified, using the following
|
||||
//! method to get WritableNeighbors if want to
|
||||
virtual const Neighbors get_neighbors(level_t level,
|
||||
node_id_t id) const override;
|
||||
const Neighbors get_neighbors(level_t level, node_id_t id) const override;
|
||||
|
||||
|
||||
//! Add vector and key to hnsw entity, and local id will be saved in id
|
||||
virtual int add_vector(level_t level, key_t key,
|
||||
const std::string &sparse_vec_buffer,
|
||||
uint32_t sparse_count, node_id_t *id) override;
|
||||
int add_vector(level_t level, key_t key, const std::string &sparse_vec_buffer,
|
||||
uint32_t sparse_count, node_id_t *id) override;
|
||||
|
||||
//! Add vector and id to hnsw entity
|
||||
virtual int add_vector_with_id(level_t level, node_id_t id,
|
||||
const std::string &sparse_vec,
|
||||
uint32_t sparse_count) override;
|
||||
int add_vector_with_id(level_t level, node_id_t id,
|
||||
const std::string &sparse_vec,
|
||||
uint32_t sparse_count) override;
|
||||
|
||||
virtual int update_neighbors(
|
||||
int update_neighbors(
|
||||
level_t level, node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
|
||||
@@ -116,13 +113,13 @@ class HnswSparseStreamerEntity : public HnswSparseEntity {
|
||||
|
||||
//! Append neighbor_id to node id neighbors on level
|
||||
//! Notice: the caller must be ensure the neighbors not full
|
||||
virtual void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
void add_neighbor(level_t level, node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
|
||||
//! Dump index by dumper
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
void update_ep_and_level(node_id_t ep, level_t level) override;
|
||||
|
||||
public:
|
||||
//! Constructor
|
||||
|
||||
@@ -28,7 +28,7 @@ class IVFBuilder : public IndexBuilder {
|
||||
IVFBuilder();
|
||||
|
||||
//! Destructor
|
||||
~IVFBuilder();
|
||||
~IVFBuilder() override;
|
||||
|
||||
//! Disable them
|
||||
IVFBuilder(const IVFBuilder &) = delete;
|
||||
@@ -36,28 +36,27 @@ class IVFBuilder : public IndexBuilder {
|
||||
|
||||
public:
|
||||
//! Initialize the builder
|
||||
virtual int init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) override;
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms) override;
|
||||
|
||||
//! Cleanup the builder
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
int train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Train the data
|
||||
virtual int train(const IndexTrainer::Pointer &trainer) override;
|
||||
int train(const IndexTrainer::Pointer &trainer) override;
|
||||
|
||||
//! Build the index
|
||||
virtual int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
int build(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) override;
|
||||
|
||||
//! Dump index into file system
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override {
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
@@ -84,25 +83,25 @@ class IVFBuilder : public IndexBuilder {
|
||||
Iterator(RandomAccessIndexHolder *owner) : holder_(owner) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {}
|
||||
~Iterator(void) override {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return holder_->element(id_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return id_ < holder_->count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return holder_->key(id_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
++id_;
|
||||
}
|
||||
|
||||
@@ -117,32 +116,32 @@ class IVFBuilder : public IndexBuilder {
|
||||
: features_(std::make_shared<CompactIndexFeatures>(meta)) {}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
virtual size_t count(void) const override {
|
||||
size_t count(void) const override {
|
||||
return features_->count();
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
virtual size_t dimension(void) const override {
|
||||
size_t dimension(void) const override {
|
||||
return features_->dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return features_->data_type();
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
size_t element_size(void) const override {
|
||||
return features_->element_size();
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
virtual bool multipass(void) const override {
|
||||
bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexHolder::Iterator::Pointer(
|
||||
new RandomAccessIndexHolder::Iterator(this));
|
||||
}
|
||||
|
||||
@@ -114,25 +114,25 @@ class IVFCentroidIndex {
|
||||
Iterator(std::vector<const void *> *features) : features_(features) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~Iterator(void) {}
|
||||
~Iterator(void) override {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return (*features_)[id_];
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return id_ < features_->size();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return id_;
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
++id_;
|
||||
}
|
||||
|
||||
@@ -168,32 +168,32 @@ class IVFCentroidIndex {
|
||||
}
|
||||
|
||||
//! Retrieve count of elements in holder (-1 indicates unknown)
|
||||
virtual size_t count(void) const override {
|
||||
size_t count(void) const override {
|
||||
return features_.size();
|
||||
}
|
||||
|
||||
//! Retrieve dimension
|
||||
virtual size_t dimension(void) const override {
|
||||
size_t dimension(void) const override {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return data_type_;
|
||||
}
|
||||
|
||||
//! Retrieve element size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
size_t element_size(void) const override {
|
||||
return element_size_;
|
||||
}
|
||||
|
||||
//! Retrieve if it can multi-pass
|
||||
virtual bool multipass(void) const override {
|
||||
bool multipass(void) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
IndexHolder::Iterator::Pointer create_iterator(void) override {
|
||||
return IndexHolder::Iterator::Pointer(
|
||||
new CentroidsIndexHolder::Iterator(&features_));
|
||||
}
|
||||
|
||||
@@ -35,37 +35,37 @@ class IVFIndexProvider : public IndexProvider {
|
||||
|
||||
public:
|
||||
//! Create a new iterator
|
||||
virtual Iterator::Pointer create_iterator(void) override {
|
||||
Iterator::Pointer create_iterator(void) override {
|
||||
return Iterator::Pointer(new (std::nothrow) SortedIterator(entity_));
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
virtual size_t count(void) const override {
|
||||
size_t count(void) const override {
|
||||
return entity_->vector_count();
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
virtual size_t dimension(void) const override {
|
||||
size_t dimension(void) const override {
|
||||
return meta_.dimension();
|
||||
}
|
||||
|
||||
//! Retrieve type of vector
|
||||
virtual IndexMeta::DataType data_type(void) const override {
|
||||
IndexMeta::DataType data_type(void) const override {
|
||||
return meta_.data_type();
|
||||
}
|
||||
|
||||
//! Retrieve vector size in bytes
|
||||
virtual size_t element_size(void) const override {
|
||||
size_t element_size(void) const override {
|
||||
return meta_.element_size();
|
||||
}
|
||||
|
||||
//! Retrieve a vector using a primary key
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
//! Retrieve the owner class
|
||||
virtual const std::string &owner_class(void) const override {
|
||||
const std::string &owner_class(void) const override {
|
||||
return owner_class_;
|
||||
}
|
||||
|
||||
@@ -88,22 +88,22 @@ class IVFIndexProvider : public IndexProvider {
|
||||
//! Retrieve pointer of data
|
||||
//! NOTICE: the vec feature will be changed after iterating to next, so
|
||||
//! the caller need to keep a copy of it before iterator to next vector
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(current_local_id());
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return pos_ < count_;
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(current_local_id());
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
++pos_;
|
||||
}
|
||||
|
||||
@@ -126,22 +126,22 @@ class IVFIndexProvider : public IndexProvider {
|
||||
Iterator(const IVFEntity::Pointer &entity) : entity_(entity) {}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
virtual const void *data(void) const override {
|
||||
const void *data(void) const override {
|
||||
return entity_->get_vector(index_);
|
||||
}
|
||||
|
||||
//! Test if the iterator is valid
|
||||
virtual bool is_valid(void) const override {
|
||||
bool is_valid(void) const override {
|
||||
return index_ < entity_->vector_count();
|
||||
}
|
||||
|
||||
//! Retrieve primary key
|
||||
virtual uint64_t key(void) const override {
|
||||
uint64_t key(void) const override {
|
||||
return entity_->get_key(index_);
|
||||
}
|
||||
|
||||
//! Next iterator
|
||||
virtual void next(void) override {
|
||||
void next(void) override {
|
||||
++index_;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,52 +26,50 @@ namespace core {
|
||||
class IVFSearcher : public IndexSearcher {
|
||||
public:
|
||||
//! Initialize Searcher
|
||||
virtual int init(const ailego::Params ¶meters) override;
|
||||
int init(const ailego::Params ¶meters) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load index from container
|
||||
virtual int load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer metric) override;
|
||||
int load(IndexStorage::Pointer container,
|
||||
IndexMetric::Pointer metric) override;
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override;
|
||||
int unload(void) override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override;
|
||||
const Stats &stats(void) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
//! Retrieve params of index
|
||||
virtual const ailego::Params ¶ms(void) const override {
|
||||
const ailego::Params ¶ms(void) const override {
|
||||
return params_;
|
||||
}
|
||||
|
||||
|
||||
@@ -30,25 +30,25 @@ class IVFSearcherContext : public IndexSearcher::Context {
|
||||
|
||||
public:
|
||||
//! Set topk of search result
|
||||
virtual void set_topk(uint32_t k) override {
|
||||
void set_topk(uint32_t k) override {
|
||||
topk_ = k;
|
||||
result_heap_.limit(topk_);
|
||||
result_heap_.set_threshold(this->threshold());
|
||||
}
|
||||
|
||||
//! Retrieve search result
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
//! Retrieve search result with index
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
ailego_assert_with(results_.size() > idx, "invalid index");
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
//! Retrieve mutable result with index
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
@@ -58,7 +58,7 @@ class IVFSearcherContext : public IndexSearcher::Context {
|
||||
}
|
||||
|
||||
//! Update the parameters of context
|
||||
virtual int update(const ailego::Params ¶ms) override {
|
||||
int update(const ailego::Params ¶ms) override {
|
||||
params.get(PARAM_IVF_SEARCHER_BRUTE_FORCE_THRESHOLD,
|
||||
&bruteforce_threshold_);
|
||||
params.get(PARAM_IVF_SEARCHER_SCAN_RATIO, &scan_ratio_);
|
||||
@@ -79,7 +79,7 @@ class IVFSearcherContext : public IndexSearcher::Context {
|
||||
}
|
||||
|
||||
//! Retrieve magic number
|
||||
virtual uint32_t magic(void) const override {
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,59 +27,57 @@ namespace core {
|
||||
class IVFStreamer : public IndexStreamer {
|
||||
public:
|
||||
//! Initialize Searcher
|
||||
virtual int init(const IndexMeta & /*meta*/,
|
||||
const ailego::Params & /*params*/) override;
|
||||
int init(const IndexMeta & /*meta*/,
|
||||
const ailego::Params & /*params*/) override;
|
||||
|
||||
//! Cleanup Searcher
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
//! Load index from container
|
||||
virtual int open(IndexStorage::Pointer storage) override;
|
||||
int open(IndexStorage::Pointer storage) override;
|
||||
|
||||
virtual int flush(uint64_t /*check_point*/) override {
|
||||
int flush(uint64_t /*check_point*/) override {
|
||||
return 0;
|
||||
}
|
||||
virtual int close(void) override {
|
||||
int close(void) override {
|
||||
return this->unload();
|
||||
}
|
||||
|
||||
//! Unload index
|
||||
virtual int unload(void) override;
|
||||
int unload(void) override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity brute force search
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
//! Similarity search
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const Stats &stats(void) const override;
|
||||
const Stats &stats(void) const override;
|
||||
|
||||
//! Create a searcher context
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
Context::Pointer create_context(void) const override;
|
||||
|
||||
//! Create a new iterator
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
//! Retrieve meta of index
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(id, block);
|
||||
}
|
||||
|
||||
|
||||
@@ -38,44 +38,44 @@ class VamanaContext : public IndexContext {
|
||||
VamanaContext(const IndexMetric::Pointer &metric,
|
||||
const VamanaEntity::Pointer &entity);
|
||||
|
||||
virtual ~VamanaContext();
|
||||
~VamanaContext() override;
|
||||
|
||||
virtual void set_topk(uint32_t val) override {
|
||||
void set_topk(uint32_t val) override {
|
||||
topk_ = val;
|
||||
topk_heap_.limit(std::max(val, ef_));
|
||||
}
|
||||
|
||||
virtual const IndexDocumentList &result(void) const override {
|
||||
const IndexDocumentList &result(void) const override {
|
||||
return results_[0];
|
||||
}
|
||||
|
||||
virtual const IndexDocumentList &result(size_t idx) const override {
|
||||
const IndexDocumentList &result(size_t idx) const override {
|
||||
return results_[idx];
|
||||
}
|
||||
|
||||
virtual IndexDocumentList *mutable_result(size_t idx) override {
|
||||
IndexDocumentList *mutable_result(size_t idx) override {
|
||||
ailego_assert_with(idx < results_.size(), "invalid idx");
|
||||
return &results_[idx];
|
||||
}
|
||||
|
||||
virtual uint32_t magic(void) const override {
|
||||
uint32_t magic(void) const override {
|
||||
return magic_;
|
||||
}
|
||||
|
||||
virtual void set_debug_mode(bool enable) override {
|
||||
void set_debug_mode(bool enable) override {
|
||||
debug_mode_ = enable;
|
||||
}
|
||||
virtual bool debug_mode(void) const override {
|
||||
bool debug_mode(void) const override {
|
||||
return debug_mode_;
|
||||
}
|
||||
|
||||
virtual std::string debug_string(void) const override {
|
||||
std::string debug_string(void) const override {
|
||||
char buf[4096];
|
||||
size_t size = snprintf(buf, sizeof(buf), "scan_cnt=%zu", get_scan_num());
|
||||
return std::string(buf, size);
|
||||
}
|
||||
|
||||
virtual int update(const ailego::Params ¶ms) override;
|
||||
int update(const ailego::Params ¶ms) override;
|
||||
|
||||
int init(ContextType type);
|
||||
|
||||
|
||||
@@ -26,89 +26,86 @@ class VamanaStreamer : public IndexStreamer {
|
||||
using ContextPointer = IndexStreamer::Context::Pointer;
|
||||
|
||||
VamanaStreamer(void);
|
||||
virtual ~VamanaStreamer(void);
|
||||
~VamanaStreamer(void) override;
|
||||
|
||||
VamanaStreamer(const VamanaStreamer &) = delete;
|
||||
VamanaStreamer &operator=(const VamanaStreamer &) = delete;
|
||||
|
||||
protected:
|
||||
virtual int init(const IndexMeta &imeta,
|
||||
const ailego::Params ¶ms) override;
|
||||
int init(const IndexMeta &imeta, const ailego::Params ¶ms) override;
|
||||
|
||||
virtual int cleanup(void) override;
|
||||
int cleanup(void) override;
|
||||
|
||||
virtual Context::Pointer create_context(void) const override;
|
||||
Context::Pointer create_context(void) const override;
|
||||
|
||||
virtual IndexProvider::Pointer create_provider(void) const override;
|
||||
IndexProvider::Pointer create_provider(void) const override;
|
||||
|
||||
virtual int add_impl(uint64_t pkey, const void *query,
|
||||
int add_impl(uint64_t pkey, const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
int add_with_id_impl(uint32_t id, const void *query,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
virtual int add_with_id_impl(uint32_t id, const void *query,
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count, Context::Pointer &context) const override;
|
||||
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) override;
|
||||
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_impl(const void *query, const IndexQueryMeta &qmeta,
|
||||
uint32_t count,
|
||||
Context::Pointer &context) const override;
|
||||
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, ContextPointer &context) const override {
|
||||
ContextPointer &context) const override {
|
||||
return search_bf_by_p_keys_impl(query, p_keys, qmeta, 1, context);
|
||||
}
|
||||
|
||||
virtual int search_bf_by_p_keys_impl(
|
||||
const void *query, const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
int search_bf_by_p_keys_impl(const void *query,
|
||||
const std::vector<std::vector<uint64_t>> &p_keys,
|
||||
const IndexQueryMeta &qmeta, uint32_t count,
|
||||
ContextPointer &context) const override;
|
||||
|
||||
virtual const void *get_vector(uint64_t key) const override {
|
||||
const void *get_vector(uint64_t key) const override {
|
||||
return entity_->get_vector_by_key(key);
|
||||
}
|
||||
|
||||
virtual int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector(const uint64_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector_by_key(key, block);
|
||||
}
|
||||
|
||||
virtual const void *get_vector_by_id(uint32_t id) const override {
|
||||
const void *get_vector_by_id(uint32_t id) const override {
|
||||
return entity_->get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_id(
|
||||
const uint32_t id, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_id(const uint32_t id,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
return entity_->get_vector(id, block);
|
||||
}
|
||||
|
||||
virtual int open(IndexStorage::Pointer stg) override;
|
||||
int open(IndexStorage::Pointer stg) override;
|
||||
|
||||
virtual int close(void) override;
|
||||
int close(void) override;
|
||||
|
||||
virtual int flush(uint64_t checkpoint) override;
|
||||
int flush(uint64_t checkpoint) override;
|
||||
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
|
||||
virtual const Stats &stats(void) const override {
|
||||
const Stats &stats(void) const override {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
virtual const IndexMeta &meta(void) const override {
|
||||
const IndexMeta &meta(void) const override {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
virtual void print_debug_info() override;
|
||||
void print_debug_info() override;
|
||||
|
||||
private:
|
||||
inline int check_params(const void *query,
|
||||
|
||||
@@ -44,28 +44,28 @@ enum class VamanaStorageMode { kMmap = 0, kBufferPool = 1, kContiguous = 2 };
|
||||
class VamanaStreamerEntity : public VamanaEntity {
|
||||
public:
|
||||
// Virtual interface implementation
|
||||
virtual int cleanup() override;
|
||||
virtual const VamanaEntity::Pointer clone() const override;
|
||||
virtual key_t get_key(node_id_t id) const override;
|
||||
virtual const void *get_vector(node_id_t id) const override;
|
||||
virtual int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
virtual int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
virtual int get_vector(
|
||||
int cleanup() override;
|
||||
const VamanaEntity::Pointer clone() const override;
|
||||
key_t get_key(node_id_t id) const override;
|
||||
const void *get_vector(node_id_t id) const override;
|
||||
int get_vector(const node_id_t id,
|
||||
IndexStorage::MemoryBlock &block) const override;
|
||||
int get_vector(const node_id_t *ids, uint32_t count,
|
||||
const void **vecs) const override;
|
||||
int get_vector(
|
||||
const node_id_t *ids, uint32_t count,
|
||||
std::vector<IndexStorage::MemoryBlock> &vec_blocks) const override;
|
||||
virtual const Neighbors get_neighbors(node_id_t id) const override;
|
||||
const Neighbors get_neighbors(node_id_t id) const override;
|
||||
|
||||
virtual int add_vector(key_t key, const void *vec, node_id_t *id) override;
|
||||
virtual int add_vector_with_id(node_id_t id, const void *vec) override;
|
||||
virtual int update_neighbors(
|
||||
int add_vector(key_t key, const void *vec, node_id_t *id) override;
|
||||
int add_vector_with_id(node_id_t id, const void *vec) override;
|
||||
int update_neighbors(
|
||||
node_id_t id,
|
||||
const std::vector<std::pair<node_id_t, dist_t>> &neighbors) override;
|
||||
virtual void add_neighbor(node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
virtual int dump(const IndexDumper::Pointer &dumper) override;
|
||||
virtual void update_entry_point(node_id_t ep) override;
|
||||
void add_neighbor(node_id_t id, uint32_t size,
|
||||
node_id_t neighbor_id) override;
|
||||
int dump(const IndexDumper::Pointer &dumper) override;
|
||||
void update_entry_point(node_id_t ep) override;
|
||||
|
||||
// --- Neighbor distance storage ---
|
||||
int ensure_dist_storage() override;
|
||||
@@ -90,13 +90,13 @@ class VamanaStreamerEntity : public VamanaEntity {
|
||||
VamanaStreamerEntity(IndexStreamer::Stats &stats);
|
||||
~VamanaStreamerEntity();
|
||||
|
||||
virtual const void *get_vector_by_key(key_t key) const override {
|
||||
const void *get_vector_by_key(key_t key) const override {
|
||||
auto id = get_id(key);
|
||||
return id == kInvalidNodeId ? nullptr : get_vector(id);
|
||||
}
|
||||
|
||||
virtual int get_vector_by_key(
|
||||
const key_t key, IndexStorage::MemoryBlock &block) const override {
|
||||
int get_vector_by_key(const key_t key,
|
||||
IndexStorage::MemoryBlock &block) const override {
|
||||
auto id = get_id(key);
|
||||
if (id != kInvalidNodeId) {
|
||||
return get_vector(id, block);
|
||||
|
||||
Reference in New Issue
Block a user