feat: add diskann index (#369)
This commit is contained in:
@@ -19,189 +19,195 @@ namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! General
|
||||
static const std::string GENERAL_CLUSTER_COUNT =
|
||||
"proxima.general.cluster.count";
|
||||
static const std::string GENERAL_CLUSTER_COUNT = "zvec.general.cluster.count";
|
||||
static const std::string GENERAL_THREAD_COUNT =
|
||||
"proxima.general.cluster.thread_count";
|
||||
"zvec.general.cluster.thread_count";
|
||||
|
||||
//! Optimize K-means
|
||||
static const std::string OPTKMEANS_CLUSTER_COUNT =
|
||||
"proxima.optkmeans.cluster.count";
|
||||
"zvec.optkmeans.cluster.count";
|
||||
static const std::string OPTKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.optkmeans.cluster.max_iterations";
|
||||
"zvec.optkmeans.cluster.max_iterations";
|
||||
static const std::string OPTKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.optkmeans.cluster.epsilon";
|
||||
"zvec.optkmeans.cluster.epsilon";
|
||||
static const std::string OPTKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.optkmeans.cluster.shard_factor";
|
||||
"zvec.optkmeans.cluster.shard_factor";
|
||||
static const std::string OPTKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.optkmeans.cluster.purge_empty";
|
||||
"zvec.optkmeans.cluster.purge_empty";
|
||||
static const std::string OPTKMEANS_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.optkmeans.cluster.markov_chain_length";
|
||||
"zvec.optkmeans.cluster.markov_chain_length";
|
||||
static const std::string OPTKMEANS_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.optkmeans.cluster.assumption_free";
|
||||
"zvec.optkmeans.cluster.assumption_free";
|
||||
|
||||
//! K-means
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "proxima.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "zvec.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON =
|
||||
"proxima.kmeans.cluster.epsilon";
|
||||
"zvec.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON = "zvec.kmeans.cluster.epsilon";
|
||||
static const std::string KMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmeans.cluster.max_iterations";
|
||||
"zvec.kmeans.cluster.max_iterations";
|
||||
static const std::string KMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "proxima.kmeans.cluster.batch";
|
||||
"zvec.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "zvec.kmeans.cluster.batch";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmeans.cluster.seeker_class";
|
||||
"zvec.kmeans.cluster.seeker_class";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmeans.cluster.seeker_params";
|
||||
"zvec.kmeans.cluster.seeker_params";
|
||||
|
||||
//! Mini Batch K-means
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.count";
|
||||
"zvec.minibatchkmeans.cluster.count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.minibatchkmeans.cluster.shard_factor";
|
||||
"zvec.minibatchkmeans.cluster.shard_factor";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.minibatchkmeans.cluster.epsilon";
|
||||
"zvec.minibatchkmeans.cluster.epsilon";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.minibatchkmeans.cluster.max_iterations";
|
||||
"zvec.minibatchkmeans.cluster.max_iterations";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.minibatchkmeans.cluster.purge_empty";
|
||||
"zvec.minibatchkmeans.cluster.purge_empty";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_TRY_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.try_count";
|
||||
"zvec.minibatchkmeans.cluster.try_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_BATCH_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.batch_count";
|
||||
"zvec.minibatchkmeans.cluster.batch_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_class";
|
||||
"zvec.minibatchkmeans.cluster.seeker_class";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_params";
|
||||
"zvec.minibatchkmeans.cluster.seeker_params";
|
||||
|
||||
//! K-means++
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT =
|
||||
"proxima.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT = "zvec.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS =
|
||||
"proxima.kmeanspp.cluster.class";
|
||||
"zvec.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS = "zvec.kmeanspp.cluster.class";
|
||||
static const std::string KMEANSPP_CLUSTER_PARAMS =
|
||||
"proxima.kmeanspp.cluster.params";
|
||||
"zvec.kmeanspp.cluster.params";
|
||||
|
||||
//! K-MC2
|
||||
static const std::string KMC2_CLUSTER_COUNT = "proxima.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_COUNT = "zvec.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmc2.cluster.shard_factor";
|
||||
"zvec.kmc2.cluster.shard_factor";
|
||||
static const std::string KMC2_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.kmc2.cluster.markov_chain_length";
|
||||
"zvec.kmc2.cluster.markov_chain_length";
|
||||
static const std::string KMC2_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "proxima.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "proxima.kmc2.cluster.params";
|
||||
"zvec.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "zvec.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "zvec.kmc2.cluster.params";
|
||||
|
||||
//! Bisecting K-means
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT =
|
||||
"proxima.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT = "zvec.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_INIT_COUNT =
|
||||
"proxima.bikmeans.cluster.init_count";
|
||||
"zvec.bikmeans.cluster.init_count";
|
||||
static const std::string BIKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.bikmeans.cluster.purge_empty";
|
||||
"zvec.bikmeans.cluster.purge_empty";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_CLASS =
|
||||
"proxima.bikmeans.cluster.first_class";
|
||||
"zvec.bikmeans.cluster.first_class";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_CLASS =
|
||||
"proxima.bikmeans.cluster.second_class";
|
||||
"zvec.bikmeans.cluster.second_class";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.bikmeans.cluster.first_params";
|
||||
"zvec.bikmeans.cluster.first_params";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.bikmeans.cluster.second_params";
|
||||
"zvec.bikmeans.cluster.second_params";
|
||||
|
||||
//! K-medoids
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT =
|
||||
"proxima.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT = "zvec.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmedoids.cluster.shard_factor";
|
||||
"zvec.kmedoids.cluster.shard_factor";
|
||||
static const std::string KMEDOIDS_CLUSTER_EPSILON =
|
||||
"proxima.kmedoids.cluster.epsilon";
|
||||
"zvec.kmedoids.cluster.epsilon";
|
||||
static const std::string KMEDOIDS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmedoids.cluster.max_iterations";
|
||||
"zvec.kmedoids.cluster.max_iterations";
|
||||
static const std::string KMEDOIDS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmedoids.cluster.purge_empty";
|
||||
"zvec.kmedoids.cluster.purge_empty";
|
||||
static const std::string KMEDOIDS_CLUSTER_BENCH_RATIO =
|
||||
"proxima.kmedoids.cluster.bench_ratio";
|
||||
"zvec.kmedoids.cluster.bench_ratio";
|
||||
static const std::string KMEDOIDS_CLUSTER_ONLY_MEANS =
|
||||
"proxima.kmedoids.cluster.only_means";
|
||||
"zvec.kmedoids.cluster.only_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_WITHOUT_MEANS =
|
||||
"proxima.kmedoids.cluster.without_means";
|
||||
"zvec.kmedoids.cluster.without_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmedoids.cluster.seeker_class";
|
||||
"zvec.kmedoids.cluster.seeker_class";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmedoids.cluster.seeker_params";
|
||||
"zvec.kmedoids.cluster.seeker_params";
|
||||
|
||||
//! Stratified
|
||||
static const std::string STRATIFIED_CLUSTER_COUNT =
|
||||
"proxima.stratified.cluster.count";
|
||||
"zvec.stratified.cluster.count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_CLASS =
|
||||
"proxima.stratified.cluster.first_class";
|
||||
"zvec.stratified.cluster.first_class";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_CLASS =
|
||||
"proxima.stratified.cluster.second_class";
|
||||
"zvec.stratified.cluster.second_class";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_COUNT =
|
||||
"proxima.stratified.cluster.first_count";
|
||||
"zvec.stratified.cluster.first_count";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_COUNT =
|
||||
"proxima.stratified.cluster.second_count";
|
||||
"zvec.stratified.cluster.second_count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.stratified.cluster.first_params";
|
||||
"zvec.stratified.cluster.first_params";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.stratified.cluster.second_params";
|
||||
"zvec.stratified.cluster.second_params";
|
||||
static const std::string STRATIFIED_CLUSTER_AUTO_TUNING =
|
||||
"proxima.stratified.cluster.auto_tuning";
|
||||
"zvec.stratified.cluster.auto_tuning";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_POOL_COUNT =
|
||||
"proxima.stratified.cluster.second_pool_count";
|
||||
"zvec.stratified.cluster.second_pool_count";
|
||||
|
||||
//! Gap Statistics
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN =
|
||||
"proxima.gapstats.cluster_estimater.k_min";
|
||||
"zvec.gapstats.cluster_estimater.k_min";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX =
|
||||
"proxima.gapstats.cluster_estimater.k_max";
|
||||
"zvec.gapstats.cluster_estimater.k_max";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_min_step";
|
||||
"zvec.gapstats.cluster_estimater.k_min_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_max_step";
|
||||
"zvec.gapstats.cluster_estimater.k_max_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_TRY_COUNT =
|
||||
"proxima.gapstats.cluster_estimater.try_count";
|
||||
"zvec.gapstats.cluster_estimater.try_count";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_SHARD_FACTOR =
|
||||
"proxima.gapstats.cluster_estimater.shard_factor";
|
||||
"zvec.gapstats.cluster_estimater.shard_factor";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_ENABLE_MC2 =
|
||||
"proxima.gapstats.cluster_estimater.enable_mc2";
|
||||
"zvec.gapstats.cluster_estimater.enable_mc2";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.gapstats.cluster_estimater.markov_chain_length";
|
||||
"zvec.gapstats.cluster_estimater.markov_chain_length";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_CLUSTER_CLASS =
|
||||
"proxima.gapstats.cluster_estimater.cluster_class";
|
||||
"zvec.gapstats.cluster_estimater.cluster_class";
|
||||
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.cluster.trainer.sample_count";
|
||||
"zvec.cluster.trainer.sample_count";
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.cluster.trainer.sample_ratio";
|
||||
"zvec.cluster.trainer.sample_ratio";
|
||||
static const std::string CLUSTER_TRAINER_THREAD_COUNT =
|
||||
"proxima.cluster.trainer.thread_count";
|
||||
"zvec.cluster.trainer.thread_count";
|
||||
static const std::string CLUSTER_TRAINER_FILE_NAME =
|
||||
"proxima.cluster.trainer.file_name";
|
||||
"zvec.cluster.trainer.file_name";
|
||||
static const std::string CLUSTER_TRAINER_CLASS_NAME =
|
||||
"proxima.cluster.trainer.class_name";
|
||||
"zvec.cluster.trainer.class_name";
|
||||
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.stratified.trainer.sample_count";
|
||||
"zvec.stratified.trainer.sample_count";
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.stratified.trainer.sample_ratio";
|
||||
"zvec.stratified.trainer.sample_ratio";
|
||||
static const std::string STRATIFIED_TRAINER_THREAD_COUNT =
|
||||
"proxima.stratified.trainer.thread_count";
|
||||
"zvec.stratified.trainer.thread_count";
|
||||
static const std::string STRATIFIED_TRAINER_FILE_NAME =
|
||||
"proxima.stratified.trainer.file_name";
|
||||
"zvec.stratified.trainer.file_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLASS_NAME =
|
||||
"proxima.stratified.trainer.class_name";
|
||||
"zvec.stratified.trainer.class_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLUSTER_COUNT =
|
||||
"proxima.stratified.trainer.cluster_count";
|
||||
"zvec.stratified.trainer.cluster_count";
|
||||
static const std::string STRATIFIED_TRAINER_AUTOAUNE =
|
||||
"proxima.stratified.trainer.autotune";
|
||||
"zvec.stratified.trainer.autotune";
|
||||
static const std::string STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX =
|
||||
"proxima.stratified.trainer.cluster_params_in_level_";
|
||||
"zvec.stratified.trainer.cluster_params_in_level_";
|
||||
|
||||
static const std::string MULTI_CHUNK_CLUSTER_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_CHUNK_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.chunk_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_THREAD_COUNT =
|
||||
"zvec.cluster.multi_chunk_cluster.thread_count";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_EPSILON =
|
||||
"zvec.cluster.multi_chunk_cluster.epsilon";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MAX_ITERATIONS =
|
||||
"zvec.cluster.multi_chunk_cluster.max_iterations";
|
||||
static const std::string MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"zvec.cluster.multi_chunk_cluster.markov_chain_length";
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
|
||||
@@ -0,0 +1,436 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "multi_chunk_cluster.h"
|
||||
#include <chrono>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
bool MultiChunkClusterAlgorithm::is_valid(void) const {
|
||||
return features_ && features_->count();
|
||||
}
|
||||
|
||||
bool MultiChunkClusterAlgorithm::check_centroids(
|
||||
const IndexCluster::CentroidList & /*cents*/) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::init_centroids(
|
||||
size_t count, IndexCluster::CentroidList *out) {
|
||||
// Just resize, because the get random centroid step is done by cluster_once
|
||||
out->resize(count);
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::reset(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::cleanup(void) {
|
||||
features_.reset();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void MultiChunkClusterAlgorithm::suggest(uint32_t k) {
|
||||
cluster_count_ = k;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::update(const ailego::Params ¶ms) {
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
int MultiChunkClusterAlgorithm::update_params(const ailego::Params ¶ms) {
|
||||
params.get(GENERAL_THREAD_COUNT, &thread_count_);
|
||||
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
|
||||
|
||||
params.get(MULTI_CHUNK_CLUSTER_THREAD_COUNT, &thread_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_CHUNK_COUNT, &chunk_count_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_EPSILON, &epsilon_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MAX_ITERATIONS, &max_iterations_);
|
||||
params.get(MULTI_CHUNK_CLUSTER_MARKOV_CHAIN_LENGTH, &markov_chain_length_);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init_distance_func() {
|
||||
IndexMetric::Pointer metric_{};
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Create metric %s failed.", meta_.metric_name().c_str());
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
int ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed wit ret %d.", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
distance_func_ = metric_->distance_matrix(1, 1);
|
||||
if (!distance_func_) {
|
||||
LOG_ERROR("DistanceMatrix function is nullptr.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::do_chunk() {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
size_t large_chunk_count = meta_.dimension() % chunk_count_;
|
||||
size_t base_chunk_dim_ = meta_.dimension() / chunk_count_;
|
||||
|
||||
chunk_dims_.clear();
|
||||
|
||||
for (size_t i = 0; i < chunk_count_; ++i) {
|
||||
if (i < large_chunk_count) {
|
||||
chunk_dims_.push_back(base_chunk_dim_ + 1);
|
||||
} else {
|
||||
chunk_dims_.push_back(base_chunk_dim_);
|
||||
}
|
||||
}
|
||||
|
||||
chunk_dim_offsets_.clear();
|
||||
chunk_dim_offsets_.push_back(0);
|
||||
for (size_t i = 1; i < chunk_count_; ++i) {
|
||||
chunk_dim_offsets_.push_back(chunk_dim_offsets_[i - 1] +
|
||||
chunk_dims_[i - 1]);
|
||||
}
|
||||
chunk_dim_offsets_.push_back(meta_.dimension());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
|
||||
int ret = update_params(params);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = do_chunk();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = init_distance_func();
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int MultiChunkClusterAlgorithm::mount(IndexFeatures::Pointer features) {
|
||||
if (!features) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!features->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
auto data_type = meta_.data_type();
|
||||
if (data_type != IndexMeta::DataType::DT_FP32 &&
|
||||
data_type != IndexMeta::DataType::DT_FP16) {
|
||||
LOG_ERROR("Unsupported meta type %u", data_type);
|
||||
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
features_ = std::move(features);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! cluster
|
||||
int MultiChunkClusterAlgorithm::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
cents.clear();
|
||||
cents.resize(chunk_count_ * cluster_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_cluster, i,
|
||||
threads->count(), ¢s, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish Chunk Count %zu, Finished Percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / chunk_count_);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkClusterAlgorithm::classify(
|
||||
IndexThreads::Pointer /*threads*/, IndexCluster::CentroidList & /*cents*/) {
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkClusterAlgorithm::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
if (chunk_count_ == 0) {
|
||||
LOG_ERROR("Invalid Chunk Count: %u", chunk_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("Invalid cluster Count: %u", cluster_count_);
|
||||
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
if (!task_group) {
|
||||
LOG_ERROR("Failed to create task group");
|
||||
return IndexError_Runtime;
|
||||
}
|
||||
|
||||
size_t features_count = features_->count();
|
||||
out->resize(features_count * chunk_count_);
|
||||
|
||||
std::atomic<size_t> finished{0};
|
||||
|
||||
for (size_t i = 0; i < threads->count(); ++i) {
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &MultiChunkClusterAlgorithm::do_label, i,
|
||||
threads->count(), cents, out, &finished));
|
||||
}
|
||||
|
||||
while (!task_group->is_finished()) {
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
cond_.wait_until(lk, std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(check_interval_secs_));
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
LOG_INFO("Finish label cnt %zu, finished percent %.3f%%", finished.load(),
|
||||
finished.load() * 100.0f / features_count);
|
||||
}
|
||||
|
||||
if (error_.load(std::memory_order_acquire)) {
|
||||
LOG_ERROR("Failed to cluster while waiting finish");
|
||||
return errcode_;
|
||||
}
|
||||
|
||||
task_group->wait_finish();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cluster
|
||||
int MultiChunkCluster::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->cluster(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Classify
|
||||
int MultiChunkCluster::classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
return algorithm_->classify(std::move(threads), cents);
|
||||
}
|
||||
|
||||
//! Label
|
||||
int MultiChunkCluster::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
return algorithm_->label(std::move(threads), cents, out);
|
||||
}
|
||||
|
||||
//! Update Cluster
|
||||
int MultiChunkCluster::update(const ailego::Params ¶ms) {
|
||||
return algorithm_->update(params);
|
||||
}
|
||||
|
||||
//! Reset Cluster
|
||||
int MultiChunkCluster::reset(void) {
|
||||
return algorithm_->reset();
|
||||
}
|
||||
|
||||
//! Cleanup Cluster
|
||||
int MultiChunkCluster::cleanup(void) {
|
||||
return algorithm_->cleanup();
|
||||
}
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void MultiChunkCluster::suggest(uint32_t k) {
|
||||
algorithm_->suggest(k);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::mount(IndexFeatures::Pointer feats) {
|
||||
return algorithm_->mount(feats);
|
||||
}
|
||||
|
||||
int MultiChunkCluster::init(const IndexMeta &meta,
|
||||
const ailego::Params ¶ms) {
|
||||
IndexMeta new_meta(meta.data_type(), meta.dimension());
|
||||
|
||||
if (meta.metric_name() == "Cosine") {
|
||||
new_meta.set_dimension(meta.dimension() - 1);
|
||||
new_meta.set_metric("InnerProduct", 0, ailego::Params());
|
||||
}
|
||||
|
||||
auto data_type = new_meta.data_type();
|
||||
|
||||
if (new_meta.metric_name() == "InnerProduct") {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(
|
||||
new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalInnerProductAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<ailego::Float16>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<float>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<double>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int8_t>);
|
||||
break;
|
||||
}
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
algorithm_.reset(new (std::nothrow)
|
||||
MultiChunkNumericalAlgorithm<int16_t>);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
LOG_ERROR("Unsupported feature types %d.", data_type);
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
algorithm_->init(new_meta, params);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,466 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <ailego/algorithm/kmeans.h>
|
||||
#include <ailego/parallel/multi_thread_list.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/core/framework/index_cluster.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_factory.h>
|
||||
#include "cluster_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! MultiChunkClusterAlgorithm
|
||||
class MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
typedef std::shared_ptr<MultiChunkClusterAlgorithm> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
virtual ~MultiChunkClusterAlgorithm() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return chunk_dims_;
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return chunk_dim_offsets_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Check Centroids
|
||||
bool check_centroids(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Test if it is valid
|
||||
bool is_valid() const;
|
||||
|
||||
//! Do chunk
|
||||
int do_chunk();
|
||||
|
||||
//! Update parameters
|
||||
int update_params(const ailego::Params ¶ms);
|
||||
|
||||
int init_distance_func();
|
||||
|
||||
//! cluster thread
|
||||
virtual void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! label thread
|
||||
virtual void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out,
|
||||
std::atomic<size_t> *finished) = 0;
|
||||
|
||||
//! Initialize Centroids
|
||||
void init_centroids(size_t count, IndexCluster::CentroidList *out);
|
||||
|
||||
private:
|
||||
constexpr static uint32_t kDefaultLogIntervalSecs = 15U;
|
||||
|
||||
protected:
|
||||
uint32_t cluster_count_{0u};
|
||||
uint32_t thread_count_{0u};
|
||||
uint32_t chunk_count_{0u};
|
||||
uint32_t max_iterations_{20u};
|
||||
bool assumption_free_{false};
|
||||
uint32_t markov_chain_length_{32};
|
||||
double epsilon_{std::numeric_limits<float>::epsilon()};
|
||||
|
||||
int errcode_{0};
|
||||
std::atomic_bool error_{false};
|
||||
uint32_t check_interval_secs_{kDefaultLogIntervalSecs};
|
||||
std::mutex mutex_{};
|
||||
std::condition_variable cond_{};
|
||||
|
||||
IndexMeta meta_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
|
||||
std::vector<uint32_t> chunk_dims_;
|
||||
std::vector<uint32_t> chunk_dim_offsets_;
|
||||
|
||||
IndexMetric::MatrixDistance distance_func_{nullptr};
|
||||
};
|
||||
|
||||
/*! Numerical cluster algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalAlgorithm : public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalKmeans<T, IndexThreads> algorithm(cluster_count_,
|
||||
chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<ailego::NumericalKmeans<T, IndexThreads>,
|
||||
IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%u) Failed to cluster.", i + 1);
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = static_cast<uint32_t>(sel_index);
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
/*! Inner Product Cluster Algorithm
|
||||
*/
|
||||
template <typename T>
|
||||
class MultiChunkNumericalInnerProductAlgorithm
|
||||
: public MultiChunkClusterAlgorithm {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(ailego::IsArithmetic<ValueType>::value,
|
||||
"ValueType must be arithmetic");
|
||||
|
||||
//! Constructor
|
||||
MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkNumericalInnerProductAlgorithm() = default;
|
||||
|
||||
protected:
|
||||
//! cluster thread
|
||||
void do_cluster(size_t idx, size_t chunk_step,
|
||||
IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished);
|
||||
|
||||
//! label thread
|
||||
void do_label(size_t idx, size_t chunk_step,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished);
|
||||
};
|
||||
|
||||
//! cluster thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_cluster(
|
||||
size_t idx, size_t chunk_step, IndexCluster::CentroidList *cents,
|
||||
std::atomic<size_t> *finished) {
|
||||
for (size_t chunk = idx; chunk < chunk_count_; chunk += chunk_step) {
|
||||
auto chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads> algorithm(
|
||||
cluster_count_, chunk_dim);
|
||||
|
||||
// mount features into algorithm
|
||||
auto features_count = features_->count();
|
||||
|
||||
algorithm.feature_matrix_reserve(features_count);
|
||||
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
auto vec = reinterpret_cast<const T *>(features_->element(i));
|
||||
algorithm.append(vec + chunk_dim_offsets_[chunk], chunk_dim);
|
||||
}
|
||||
|
||||
IndexThreads::Pointer local_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
if (!local_threads) {
|
||||
error_ = IndexError_NoMemory;
|
||||
return;
|
||||
}
|
||||
|
||||
ailego::Kmc2CentroidsGenerator<
|
||||
ailego::NumericalInnerProductKmeans<T, IndexThreads>, IndexThreads>
|
||||
cent_gen;
|
||||
cent_gen.set_chain_length(markov_chain_length_);
|
||||
cent_gen.set_assumption_free(assumption_free_);
|
||||
cent_gen(&algorithm, *local_threads);
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
for (uint32_t i = 0; i < max_iterations_; ++i) {
|
||||
double old_cost, new_epsilon;
|
||||
old_cost = cost;
|
||||
|
||||
bool result = algorithm.cluster_once(*local_threads, &cost);
|
||||
if (result != true) {
|
||||
LOG_ERROR("(%zu) Failed to cluster.", (size_t)(i + 1));
|
||||
errcode_ = -1;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
new_epsilon = std::abs(cost - old_cost);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto &chunk_cents = algorithm.centroids();
|
||||
|
||||
for (size_t i = 0; i < chunk_cents.count(); ++i) {
|
||||
size_t global_cent_idx = chunk * cluster_count_ + i;
|
||||
|
||||
IndexCluster::Centroid *centroid = &(cents->at(global_cent_idx));
|
||||
centroid->set_score(algorithm.context().clusters()[i].cost());
|
||||
centroid->set_follows(algorithm.context().clusters()[i].count());
|
||||
centroid->set_feature(algorithm.centroids()[i],
|
||||
chunk_dim * meta_.unit_size());
|
||||
}
|
||||
|
||||
LOG_INFO("(%zu) Chunk Done. Clusters Count: %zu, Features: %zu, Cost: %f",
|
||||
chunk, algorithm.centroids().count(), features_->count(), cost);
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! label thread
|
||||
template <typename T>
|
||||
void MultiChunkNumericalInnerProductAlgorithm<T>::do_label(
|
||||
size_t idx, size_t step, const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out, std::atomic<size_t> *finished) {
|
||||
for (size_t id = idx; id < features_->count(); id += step) {
|
||||
const T *feat = reinterpret_cast<const T *>(features_->element(id));
|
||||
|
||||
for (size_t chunk = 0; chunk < chunk_count_; ++chunk) {
|
||||
size_t chunk_dim_offset = chunk_dim_offsets_[chunk];
|
||||
size_t chunk_dim = chunk_dims_[chunk];
|
||||
|
||||
uint32_t sel_index = 0;
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
|
||||
for (uint32_t cluster = 0; cluster < cluster_count_; ++cluster) {
|
||||
float score{0.0};
|
||||
|
||||
distance_func_(cents[chunk * cluster_count_ + cluster].feature(),
|
||||
feat + chunk_dim_offset, chunk_dim, &score);
|
||||
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_index = cluster;
|
||||
}
|
||||
}
|
||||
|
||||
(*out)[id * chunk_count_ + chunk] = sel_index;
|
||||
}
|
||||
|
||||
(*finished)++;
|
||||
}
|
||||
}
|
||||
|
||||
//! MultiChunkCluster
|
||||
class MultiChunkCluster {
|
||||
public:
|
||||
std::shared_ptr<MultiChunkCluster> Pointer;
|
||||
|
||||
//! Constructor
|
||||
MultiChunkCluster() = default;
|
||||
|
||||
//! Destructor
|
||||
~MultiChunkCluster() = default;
|
||||
|
||||
//! Initialize Cluster
|
||||
int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
int cleanup();
|
||||
|
||||
//! Reset Cluster
|
||||
int reset();
|
||||
|
||||
//! Update Cluster
|
||||
int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
int cluster(IndexThreads::Pointer threads, IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
const std::vector<uint32_t> &chunk_dims() const {
|
||||
return algorithm_->chunk_dims();
|
||||
}
|
||||
|
||||
const std::vector<uint32_t> &chunk_dim_offsets() const {
|
||||
return algorithm_->chunk_dim_offsets();
|
||||
}
|
||||
|
||||
protected:
|
||||
MultiChunkClusterAlgorithm::Pointer algorithm_{};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user