Initial commit
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
cc_library(
|
||||
NAME core_knn_cluster STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS zvec_ailego core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core ${PROJECT_ROOT_DIR}/src/core/cluster
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
//! General
|
||||
static const std::string GENERAL_CLUSTER_COUNT =
|
||||
"proxima.general.cluster.count";
|
||||
static const std::string GENERAL_THREAD_COUNT =
|
||||
"proxima.general.cluster.thread_count";
|
||||
|
||||
//! Optimize K-means
|
||||
static const std::string OPTKMEANS_CLUSTER_COUNT =
|
||||
"proxima.optkmeans.cluster.count";
|
||||
static const std::string OPTKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.optkmeans.cluster.max_iterations";
|
||||
static const std::string OPTKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.optkmeans.cluster.epsilon";
|
||||
static const std::string OPTKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.optkmeans.cluster.shard_factor";
|
||||
static const std::string OPTKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.optkmeans.cluster.purge_empty";
|
||||
static const std::string OPTKMEANS_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.optkmeans.cluster.markov_chain_length";
|
||||
static const std::string OPTKMEANS_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.optkmeans.cluster.assumption_free";
|
||||
|
||||
//! K-means
|
||||
static const std::string KMEANS_CLUSTER_COUNT = "proxima.kmeans.cluster.count";
|
||||
static const std::string KMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeans.cluster.shard_factor";
|
||||
static const std::string KMEANS_CLUSTER_EPSILON =
|
||||
"proxima.kmeans.cluster.epsilon";
|
||||
static const std::string KMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmeans.cluster.max_iterations";
|
||||
static const std::string KMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmeans.cluster.purge_empty";
|
||||
static const std::string KMEANS_CLUSTER_BATCH = "proxima.kmeans.cluster.batch";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmeans.cluster.seeker_class";
|
||||
static const std::string KMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmeans.cluster.seeker_params";
|
||||
|
||||
//! Mini Batch K-means
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.minibatchkmeans.cluster.shard_factor";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_EPSILON =
|
||||
"proxima.minibatchkmeans.cluster.epsilon";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.minibatchkmeans.cluster.max_iterations";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.minibatchkmeans.cluster.purge_empty";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_TRY_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.try_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_BATCH_COUNT =
|
||||
"proxima.minibatchkmeans.cluster.batch_count";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_class";
|
||||
static const std::string MINIBATCHKMEANS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.minibatchkmeans.cluster.seeker_params";
|
||||
|
||||
//! K-means++
|
||||
static const std::string KMEANSPP_CLUSTER_COUNT =
|
||||
"proxima.kmeanspp.cluster.count";
|
||||
static const std::string KMEANSPP_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmeanspp.cluster.shard_factor";
|
||||
static const std::string KMEANSPP_CLUSTER_CLASS =
|
||||
"proxima.kmeanspp.cluster.class";
|
||||
static const std::string KMEANSPP_CLUSTER_PARAMS =
|
||||
"proxima.kmeanspp.cluster.params";
|
||||
|
||||
//! K-MC2
|
||||
static const std::string KMC2_CLUSTER_COUNT = "proxima.kmc2.cluster.count";
|
||||
static const std::string KMC2_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmc2.cluster.shard_factor";
|
||||
static const std::string KMC2_CLUSTER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.kmc2.cluster.markov_chain_length";
|
||||
static const std::string KMC2_CLUSTER_ASSUMPTION_FREE =
|
||||
"proxima.kmc2.cluster.assumption_free";
|
||||
static const std::string KMC2_CLUSTER_CLASS = "proxima.kmc2.cluster.class";
|
||||
static const std::string KMC2_CLUSTER_PARAMS = "proxima.kmc2.cluster.params";
|
||||
|
||||
//! Bisecting K-means
|
||||
static const std::string BIKMEANS_CLUSTER_COUNT =
|
||||
"proxima.bikmeans.cluster.count";
|
||||
static const std::string BIKMEANS_CLUSTER_INIT_COUNT =
|
||||
"proxima.bikmeans.cluster.init_count";
|
||||
static const std::string BIKMEANS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.bikmeans.cluster.purge_empty";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_CLASS =
|
||||
"proxima.bikmeans.cluster.first_class";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_CLASS =
|
||||
"proxima.bikmeans.cluster.second_class";
|
||||
static const std::string BIKMEANS_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.bikmeans.cluster.first_params";
|
||||
static const std::string BIKMEANS_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.bikmeans.cluster.second_params";
|
||||
|
||||
//! K-medoids
|
||||
static const std::string KMEDOIDS_CLUSTER_COUNT =
|
||||
"proxima.kmedoids.cluster.count";
|
||||
static const std::string KMEDOIDS_CLUSTER_SHARD_FACTOR =
|
||||
"proxima.kmedoids.cluster.shard_factor";
|
||||
static const std::string KMEDOIDS_CLUSTER_EPSILON =
|
||||
"proxima.kmedoids.cluster.epsilon";
|
||||
static const std::string KMEDOIDS_CLUSTER_MAX_ITERATIONS =
|
||||
"proxima.kmedoids.cluster.max_iterations";
|
||||
static const std::string KMEDOIDS_CLUSTER_PURGE_EMPTY =
|
||||
"proxima.kmedoids.cluster.purge_empty";
|
||||
static const std::string KMEDOIDS_CLUSTER_BENCH_RATIO =
|
||||
"proxima.kmedoids.cluster.bench_ratio";
|
||||
static const std::string KMEDOIDS_CLUSTER_ONLY_MEANS =
|
||||
"proxima.kmedoids.cluster.only_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_WITHOUT_MEANS =
|
||||
"proxima.kmedoids.cluster.without_means";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_CLASS =
|
||||
"proxima.kmedoids.cluster.seeker_class";
|
||||
static const std::string KMEDOIDS_CLUSTER_SEEKER_PARAMS =
|
||||
"proxima.kmedoids.cluster.seeker_params";
|
||||
|
||||
//! Stratified
|
||||
static const std::string STRATIFIED_CLUSTER_COUNT =
|
||||
"proxima.stratified.cluster.count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_CLASS =
|
||||
"proxima.stratified.cluster.first_class";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_CLASS =
|
||||
"proxima.stratified.cluster.second_class";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_COUNT =
|
||||
"proxima.stratified.cluster.first_count";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_COUNT =
|
||||
"proxima.stratified.cluster.second_count";
|
||||
static const std::string STRATIFIED_CLUSTER_FIRST_PARAMS =
|
||||
"proxima.stratified.cluster.first_params";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_PARAMS =
|
||||
"proxima.stratified.cluster.second_params";
|
||||
static const std::string STRATIFIED_CLUSTER_AUTO_TUNING =
|
||||
"proxima.stratified.cluster.auto_tuning";
|
||||
static const std::string STRATIFIED_CLUSTER_SECOND_POOL_COUNT =
|
||||
"proxima.stratified.cluster.second_pool_count";
|
||||
|
||||
//! Gap Statistics
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN =
|
||||
"proxima.gapstats.cluster_estimater.k_min";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX =
|
||||
"proxima.gapstats.cluster_estimater.k_max";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MIN_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_min_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_K_MAX_STEP =
|
||||
"proxima.gapstats.cluster_estimater.k_max_step";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_TRY_COUNT =
|
||||
"proxima.gapstats.cluster_estimater.try_count";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_SHARD_FACTOR =
|
||||
"proxima.gapstats.cluster_estimater.shard_factor";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_ENABLE_MC2 =
|
||||
"proxima.gapstats.cluster_estimater.enable_mc2";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_MARKOV_CHAIN_LENGTH =
|
||||
"proxima.gapstats.cluster_estimater.markov_chain_length";
|
||||
static const std::string GAPSTATS_CLUSTER_ESTIMATER_CLUSTER_CLASS =
|
||||
"proxima.gapstats.cluster_estimater.cluster_class";
|
||||
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.cluster.trainer.sample_count";
|
||||
static const std::string CLUSTER_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.cluster.trainer.sample_ratio";
|
||||
static const std::string CLUSTER_TRAINER_THREAD_COUNT =
|
||||
"proxima.cluster.trainer.thread_count";
|
||||
static const std::string CLUSTER_TRAINER_FILE_NAME =
|
||||
"proxima.cluster.trainer.file_name";
|
||||
static const std::string CLUSTER_TRAINER_CLASS_NAME =
|
||||
"proxima.cluster.trainer.class_name";
|
||||
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_COUNT =
|
||||
"proxima.stratified.trainer.sample_count";
|
||||
static const std::string STRATIFIED_TRAINER_SAMPLE_RATIO =
|
||||
"proxima.stratified.trainer.sample_ratio";
|
||||
static const std::string STRATIFIED_TRAINER_THREAD_COUNT =
|
||||
"proxima.stratified.trainer.thread_count";
|
||||
static const std::string STRATIFIED_TRAINER_FILE_NAME =
|
||||
"proxima.stratified.trainer.file_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLASS_NAME =
|
||||
"proxima.stratified.trainer.class_name";
|
||||
static const std::string STRATIFIED_TRAINER_CLUSTER_COUNT =
|
||||
"proxima.stratified.trainer.cluster_count";
|
||||
static const std::string STRATIFIED_TRAINER_AUTOAUNE =
|
||||
"proxima.stratified.trainer.autotune";
|
||||
static const std::string STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX =
|
||||
"proxima.stratified.trainer.cluster_params_in_level_";
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,915 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <ailego/container/reservoir.h>
|
||||
#include <ailego/utility/float_helper.h>
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include "framework/index_cluster.h"
|
||||
#include "framework/index_error.h"
|
||||
#include "framework/index_factory.h"
|
||||
#include "cluster_params.h"
|
||||
#include "linear_seeker.h"
|
||||
#include "vector_mean.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Kmeans Cluster
|
||||
*/
|
||||
class KmeansCluster : public IndexCluster {
|
||||
public:
|
||||
//! Constructor
|
||||
KmeansCluster(void) {}
|
||||
|
||||
//! Constructor
|
||||
KmeansCluster(size_t iters, bool batch)
|
||||
: max_iterations_(iters), batch_(batch) {}
|
||||
|
||||
//! Constructor
|
||||
KmeansCluster(bool batch) : batch_(batch) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~KmeansCluster(void) {}
|
||||
|
||||
//! Initialize Cluster
|
||||
virtual int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Cluster
|
||||
virtual int cleanup(void);
|
||||
|
||||
//! Reset Cluster
|
||||
virtual int reset(void);
|
||||
|
||||
//! Update Cluster
|
||||
virtual int update(const ailego::Params ¶ms);
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
virtual void suggest(uint32_t k);
|
||||
|
||||
//! Mount features
|
||||
virtual int mount(IndexFeatures::Pointer feats);
|
||||
|
||||
//! Cluster
|
||||
virtual int cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
virtual int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
virtual int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
protected:
|
||||
//! Test if it is valid
|
||||
bool is_valid(void) const;
|
||||
|
||||
//! Cluster once
|
||||
int clustering(IndexThreads *threads, IndexCluster::CentroidList ¢s,
|
||||
double *cost);
|
||||
|
||||
//! Update parameters
|
||||
void update_params(const ailego::Params ¶ms);
|
||||
|
||||
//! Init seeker
|
||||
int init_seeker(void);
|
||||
|
||||
//! Build seeker
|
||||
int build_seeker(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Check Centroids
|
||||
bool check_centroids(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Initialize Centroids
|
||||
void init_centroids(size_t count, IndexCluster::CentroidList *out);
|
||||
|
||||
//! Initialize Shard Containers
|
||||
void init_containers(size_t shard_count);
|
||||
|
||||
//! Initialize Shard Features Containers
|
||||
void init_features_containers(size_t shard_count);
|
||||
|
||||
//! Split Clusters
|
||||
void split_clusters(IndexThreads *threads,
|
||||
const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Update Centroids
|
||||
void update_centroids(IndexThreads *threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Update Clusters
|
||||
void update_clusters(IndexThreads *threads,
|
||||
const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Update Clusters' Features
|
||||
void update_features(IndexThreads *threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Update Labels
|
||||
void update_labels(IndexThreads *threads, std::vector<uint32_t> *labels);
|
||||
|
||||
//! Split Clusters in Thread
|
||||
void split_clusters_thread(size_t index_begin, size_t index_end,
|
||||
const IndexThreads *threads);
|
||||
|
||||
//! Update Centroid in Thread
|
||||
void update_centroid_thread(size_t column, IndexCluster::CentroidList *out);
|
||||
|
||||
//! Update Cluster in Thread
|
||||
void update_cluster_thread(size_t index_begin, size_t index_end,
|
||||
const IndexThreads *threads);
|
||||
|
||||
//! Update Cluster's Features in Thread
|
||||
void update_features_thread(size_t column, IndexCluster::CentroidList *out);
|
||||
|
||||
//! Update Labels in Thread
|
||||
void update_labels_thread(size_t index_begin, size_t index_end,
|
||||
std::vector<uint32_t> *labels);
|
||||
|
||||
protected:
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
LinearSeeker::Pointer seeker_{};
|
||||
std::vector<double> shard_cluster_scores_{};
|
||||
std::vector<std::vector<const void *>> shard_cluster_features_{};
|
||||
std::shared_ptr<VectorMeanArray> shard_cluster_means_{};
|
||||
std::shared_ptr<VectorMeanArray> batch_means_{};
|
||||
std::vector<double> batch_scores_{};
|
||||
double epsilon_{std::numeric_limits<float>::epsilon()};
|
||||
float shard_factor_{16.0f};
|
||||
uint32_t max_iterations_{20u};
|
||||
uint32_t cluster_count_{0u};
|
||||
uint32_t thread_count_{0u};
|
||||
bool batch_{false};
|
||||
bool purge_empty_{false};
|
||||
};
|
||||
|
||||
/*! Centroid Features
|
||||
*/
|
||||
class KmeansCentroidFeatures : public IndexFeatures {
|
||||
public:
|
||||
//! Constructor
|
||||
KmeansCentroidFeatures(const IndexMeta &meta,
|
||||
const IndexCluster::CentroidList ¢s)
|
||||
: centroids_(cents),
|
||||
feature_size_(meta.element_size()),
|
||||
feature_dimension_(meta.dimension()),
|
||||
data_type_(meta.data_type()) {}
|
||||
|
||||
virtual size_t count(void) const {
|
||||
return centroids_.size();
|
||||
}
|
||||
|
||||
virtual size_t dimension(void) const {
|
||||
return feature_dimension_;
|
||||
}
|
||||
|
||||
virtual const void *element(size_t i) const {
|
||||
return centroids_[i].feature();
|
||||
}
|
||||
|
||||
virtual IndexMeta::DataType data_type(void) const {
|
||||
return data_type_;
|
||||
}
|
||||
|
||||
virtual size_t element_size(void) const {
|
||||
return feature_size_;
|
||||
}
|
||||
|
||||
private:
|
||||
const IndexCluster::CentroidList ¢roids_;
|
||||
size_t feature_size_;
|
||||
size_t feature_dimension_;
|
||||
IndexMeta::DataType data_type_;
|
||||
};
|
||||
|
||||
static inline std::shared_ptr<VectorMean> NewVectorMean(const IndexMeta &meta) {
|
||||
switch (meta.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
return std::make_shared<NumericalVectorMean<ailego::Float16>>(
|
||||
meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
return std::make_shared<NumericalVectorMean<float>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_FP64:
|
||||
return std::make_shared<NumericalVectorMean<double>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT8:
|
||||
return std::make_shared<NumericalVectorMean<int8_t>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT4:
|
||||
return std::make_shared<NibbleVectorMean<uint8_t>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT16:
|
||||
return std::make_shared<NumericalVectorMean<int16_t>>(meta.dimension());
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
// As binary default
|
||||
return std::make_shared<BinaryVectorMean>(meta.dimension());
|
||||
}
|
||||
|
||||
static inline std::shared_ptr<VectorMeanArray> NewVectorMeanArray(
|
||||
const IndexMeta &meta) {
|
||||
switch (meta.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP16:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<ailego::Float16>>>(
|
||||
meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_FP32:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<float>>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_FP64:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<double>>>(
|
||||
meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT8:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<int8_t>>>(
|
||||
meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT4:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NibbleVectorMean<uint8_t>>>(meta.dimension());
|
||||
|
||||
case IndexMeta::DataType::DT_INT16:
|
||||
return std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<int16_t>>>(
|
||||
meta.dimension());
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
// As binary default
|
||||
return std::make_shared<GeneralVectorMeanArray<BinaryVectorMean>>(
|
||||
meta.dimension());
|
||||
}
|
||||
|
||||
static inline std::shared_ptr<VectorMeanArray> NewVectorMeanArray(
|
||||
const IndexMeta &meta, const IndexCluster::CentroidList ¢s) {
|
||||
switch (meta.data_type()) {
|
||||
case IndexMeta::DataType::DT_FP16: {
|
||||
auto ptr = std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<ailego::Float16>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const ailego::Float16 *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
case IndexMeta::DataType::DT_FP32: {
|
||||
auto ptr =
|
||||
std::make_shared<GeneralVectorMeanArray<NumericalVectorMean<float>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const float *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
case IndexMeta::DataType::DT_FP64: {
|
||||
auto ptr =
|
||||
std::make_shared<GeneralVectorMeanArray<NumericalVectorMean<double>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const double *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
case IndexMeta::DataType::DT_INT8: {
|
||||
auto ptr =
|
||||
std::make_shared<GeneralVectorMeanArray<NumericalVectorMean<int8_t>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const int8_t *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
case IndexMeta::DataType::DT_INT4: {
|
||||
auto ptr =
|
||||
std::make_shared<GeneralVectorMeanArray<NibbleVectorMean<uint8_t>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const uint8_t *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
case IndexMeta::DataType::DT_INT16: {
|
||||
auto ptr = std::make_shared<
|
||||
GeneralVectorMeanArray<NumericalVectorMean<int16_t>>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(reinterpret_cast<const int16_t *>(it.feature()),
|
||||
meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
// As binary default
|
||||
auto ptr = std::make_shared<GeneralVectorMeanArray<BinaryVectorMean>>(
|
||||
meta.dimension());
|
||||
|
||||
for (const auto &it : cents) {
|
||||
ptr->emplace(it.feature(), meta.dimension(), it.follows());
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static inline double CalculateSSE(const IndexCluster::CentroidList ¢s) {
|
||||
double accum = 0.0;
|
||||
for (const auto &it : cents) {
|
||||
accum += it.score();
|
||||
}
|
||||
return accum;
|
||||
}
|
||||
|
||||
static inline void PurgeCentroids(IndexCluster::CentroidList ¢s,
|
||||
bool cutting) {
|
||||
size_t index = 0;
|
||||
size_t tamp = cents.size();
|
||||
|
||||
while (index < tamp) {
|
||||
if (cents[index].follows() == 0) {
|
||||
size_t last_index = tamp - 1;
|
||||
|
||||
if (index != last_index) {
|
||||
std::swap(cents[index], cents[last_index]);
|
||||
}
|
||||
tamp = last_index;
|
||||
continue;
|
||||
}
|
||||
++index;
|
||||
}
|
||||
if (cutting) {
|
||||
cents.resize(tamp);
|
||||
}
|
||||
}
|
||||
|
||||
int KmeansCluster::init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
this->update_params(params);
|
||||
|
||||
return this->init_seeker();
|
||||
}
|
||||
|
||||
int KmeansCluster::cleanup(void) {
|
||||
features_.reset();
|
||||
shard_cluster_scores_.clear();
|
||||
shard_cluster_features_.clear();
|
||||
shard_cluster_means_.reset();
|
||||
batch_means_.reset();
|
||||
batch_scores_.clear();
|
||||
seeker_->cleanup();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::reset(void) {
|
||||
features_.reset();
|
||||
shard_cluster_scores_.clear();
|
||||
shard_cluster_features_.clear();
|
||||
shard_cluster_means_->clear();
|
||||
batch_means_->clear();
|
||||
batch_scores_.clear();
|
||||
seeker_->reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::update(const ailego::Params ¶ms) {
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void KmeansCluster::suggest(uint32_t k) {
|
||||
cluster_count_ = k;
|
||||
}
|
||||
|
||||
int KmeansCluster::mount(IndexFeatures::Pointer feats) {
|
||||
if (!feats) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!feats->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
|
||||
// Check dimension
|
||||
auto data_type = meta_.data_type();
|
||||
switch (data_type) {
|
||||
case IndexMeta::DataType::DT_INT4:
|
||||
if (feats->dimension() % 2 != 0) {
|
||||
LOG_ERROR(
|
||||
"Unsupported feature dimension %zu (dimension of int4 "
|
||||
"must be an integer multiple of 2).",
|
||||
feats->dimension());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
break;
|
||||
case IndexMeta::DataType::DT_BINARY32:
|
||||
if (feats->dimension() % 32 != 0) {
|
||||
LOG_ERROR(
|
||||
"Unsupported feature dimension %zu (dimension of binary32 "
|
||||
"must be an integer multiple of 32).",
|
||||
feats->dimension());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
break;
|
||||
case IndexMeta::DataType::DT_BINARY64:
|
||||
if (feats->dimension() % 64 != 0) {
|
||||
LOG_ERROR(
|
||||
"Unsupported feature dimension %zu (dimension of binary64 "
|
||||
"must be an integer multiple of 64).",
|
||||
feats->dimension());
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
features_ = std::move(feats);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
ailego::ElapsedTime stamp;
|
||||
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
if (cents.empty()) {
|
||||
if (cluster_count_ == 0) {
|
||||
LOG_ERROR("The count of cluster is unknown.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
this->init_centroids(cluster_count_, ¢s);
|
||||
}
|
||||
|
||||
if (batch_) {
|
||||
batch_means_ = NewVectorMeanArray(meta_, cents);
|
||||
batch_scores_.clear();
|
||||
for (const auto &it : cents) {
|
||||
batch_scores_.push_back(it.score());
|
||||
}
|
||||
}
|
||||
|
||||
double cost = 0.0;
|
||||
|
||||
// we need to do clustering and update the centroids' follows, even if
|
||||
// cents.size() == 1. Otherwise, the centroid with empty follows will be
|
||||
// removed if purge_empty enabled
|
||||
for (uint32_t i = 0; (i < max_iterations_) && (cents.size() > 0); ++i) {
|
||||
double new_cost, new_epsilon;
|
||||
|
||||
int result = this->clustering(threads.get(), cents, &new_cost);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("(%u) Failed to cluster.", i + 1);
|
||||
return result;
|
||||
}
|
||||
|
||||
new_epsilon = new_cost - cost;
|
||||
LOG_DEBUG("(%u) Updated %zu Clusters, %zu Features: %zu ms, %f -> %f = %f",
|
||||
i, cents.size(), features_->count(),
|
||||
(size_t)stamp.milli_seconds(), cost, new_cost, new_epsilon);
|
||||
stamp.reset();
|
||||
|
||||
new_epsilon = std::abs(new_epsilon);
|
||||
if (new_epsilon < epsilon_) {
|
||||
break;
|
||||
}
|
||||
cost = new_cost;
|
||||
}
|
||||
|
||||
// Purge the empty centroids
|
||||
PurgeCentroids(cents, purge_empty_);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
int result = this->build_seeker(cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to build the seeker.");
|
||||
return result;
|
||||
}
|
||||
|
||||
this->update_clusters(threads.get(), cents);
|
||||
this->update_features(threads.get(), cents);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
int result = this->build_seeker(cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to build the seeker.");
|
||||
return result;
|
||||
}
|
||||
|
||||
this->update_labels(threads.get(), out);
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool KmeansCluster::is_valid(void) const {
|
||||
if (!seeker_ || !features_ || !features_->count()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
int KmeansCluster::clustering(IndexThreads *threads,
|
||||
IndexCluster::CentroidList ¢s, double *cost) {
|
||||
int result = this->build_seeker(cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to build the seeker.");
|
||||
return result;
|
||||
}
|
||||
|
||||
this->split_clusters(threads, cents);
|
||||
this->update_centroids(threads, cents);
|
||||
*cost = CalculateSSE(cents);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void KmeansCluster::update_params(const ailego::Params ¶ms) {
|
||||
params.get(GENERAL_THREAD_COUNT, &thread_count_);
|
||||
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(KMEANS_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(KMEANS_CLUSTER_SHARD_FACTOR, &shard_factor_);
|
||||
params.get(KMEANS_CLUSTER_EPSILON, &epsilon_);
|
||||
params.get(KMEANS_CLUSTER_MAX_ITERATIONS, &max_iterations_);
|
||||
params.get(KMEANS_CLUSTER_BATCH, &batch_);
|
||||
params.get(KMEANS_CLUSTER_PURGE_EMPTY, &purge_empty_);
|
||||
}
|
||||
|
||||
int KmeansCluster::init_seeker(void) {
|
||||
seeker_.reset(new (std::nothrow) LinearSeeker);
|
||||
if (!seeker_) {
|
||||
LOG_ERROR("Failed to create linear seeker.");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
int result = seeker_->init(meta_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize linear seeker.");
|
||||
return result;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int KmeansCluster::build_seeker(const IndexCluster::CentroidList ¢s) {
|
||||
int result =
|
||||
seeker_->mount(std::make_shared<KmeansCentroidFeatures>(meta_, cents));
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features for linear seeker.");
|
||||
return result;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool KmeansCluster::check_centroids(const IndexCluster::CentroidList ¢s) {
|
||||
for (const auto &it : cents) {
|
||||
if (it.size() != meta_.element_size()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void KmeansCluster::init_centroids(size_t count,
|
||||
IndexCluster::CentroidList *out) {
|
||||
size_t feature_size = features_->element_size();
|
||||
size_t features_count = features_->count();
|
||||
size_t sample_count = std::min<size_t>(count, features_count);
|
||||
|
||||
ailego::Reservoir<size_t> sampler(sample_count);
|
||||
for (size_t i = 0; i < features_count; ++i) {
|
||||
sampler.fill(i);
|
||||
}
|
||||
|
||||
// Save centroids
|
||||
out->reserve(sampler.pool().size());
|
||||
for (auto i : sampler.pool()) {
|
||||
out->emplace_back(features_->element(i), feature_size);
|
||||
}
|
||||
}
|
||||
|
||||
void KmeansCluster::init_containers(size_t shard_count) {
|
||||
if (!shard_cluster_means_) {
|
||||
shard_cluster_means_ = NewVectorMeanArray(meta_);
|
||||
}
|
||||
shard_cluster_means_->clear();
|
||||
shard_cluster_means_->resize(shard_count);
|
||||
shard_cluster_scores_.clear();
|
||||
shard_cluster_scores_.resize(shard_count);
|
||||
}
|
||||
|
||||
void KmeansCluster::init_features_containers(size_t shard_count) {
|
||||
shard_cluster_features_.resize(shard_count);
|
||||
for (auto &features : shard_cluster_features_) {
|
||||
features.clear();
|
||||
}
|
||||
}
|
||||
|
||||
void KmeansCluster::split_clusters(IndexThreads *threads,
|
||||
const IndexCluster::CentroidList ¢s) {
|
||||
// Initilize containers
|
||||
this->init_containers(threads->count() * cents.size());
|
||||
auto task_group = threads->make_group();
|
||||
|
||||
// Initilize base information
|
||||
size_t features_count = features_->count();
|
||||
size_t shard_count = std::max<size_t>(
|
||||
static_cast<size_t>(std::ceil(threads->count() * shard_factor_)), 1u);
|
||||
size_t fregment_count = (features_count + shard_count - 1) / shard_count;
|
||||
|
||||
for (size_t i = 0, index = 0; (i != shard_count) && (index < features_count);
|
||||
++i) {
|
||||
size_t next_index = index + fregment_count;
|
||||
if (next_index > features_count) {
|
||||
next_index = features_count;
|
||||
}
|
||||
|
||||
// Process in work thread
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &KmeansCluster::split_clusters_thread, index,
|
||||
next_index, threads));
|
||||
|
||||
// Next index
|
||||
index = next_index;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
}
|
||||
|
||||
void KmeansCluster::update_centroids(IndexThreads *threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
auto task_group = threads->make_group();
|
||||
for (size_t i = 0; i < cents.size(); ++i) {
|
||||
task_group->submit(ailego::Closure::New(
|
||||
this, &KmeansCluster::update_centroid_thread, i, ¢s));
|
||||
}
|
||||
task_group->wait_finish();
|
||||
}
|
||||
|
||||
void KmeansCluster::update_clusters(IndexThreads *threads,
|
||||
const IndexCluster::CentroidList ¢s) {
|
||||
// Initilize containers
|
||||
this->init_features_containers(threads->count() * cents.size());
|
||||
auto task_group = threads->make_group();
|
||||
|
||||
size_t features_count = features_->count();
|
||||
size_t shard_count = std::max<size_t>(
|
||||
static_cast<size_t>(std::ceil(threads->count() * shard_factor_)), 1u);
|
||||
size_t fregment_count = (features_count + shard_count - 1) / shard_count;
|
||||
|
||||
for (size_t i = 0, index = 0; (i != shard_count) && (index < features_count);
|
||||
++i) {
|
||||
size_t next_index = index + fregment_count;
|
||||
if (next_index > features_count) {
|
||||
next_index = features_count;
|
||||
}
|
||||
// Process in work thread
|
||||
task_group->submit(
|
||||
ailego::Closure::New(this, &KmeansCluster::update_cluster_thread, index,
|
||||
next_index, threads));
|
||||
|
||||
// Next index
|
||||
index = next_index;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
}
|
||||
|
||||
void KmeansCluster::update_features(IndexThreads *threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
auto task_group = threads->make_group();
|
||||
for (size_t i = 0; i < cents.size(); ++i) {
|
||||
// Process in work thread
|
||||
task_group->submit(ailego::Closure::New(
|
||||
this, &KmeansCluster::update_features_thread, i, ¢s));
|
||||
}
|
||||
task_group->wait_finish();
|
||||
}
|
||||
|
||||
void KmeansCluster::update_labels(IndexThreads *threads,
|
||||
std::vector<uint32_t> *labels) {
|
||||
size_t features_count = features_->count();
|
||||
size_t shard_count = std::max<size_t>(
|
||||
static_cast<size_t>(std::ceil(threads->count() * shard_factor_)), 1u);
|
||||
size_t fregment_count = (features_count + shard_count - 1) / shard_count;
|
||||
auto task_group = threads->make_group();
|
||||
|
||||
// Prepare buffer
|
||||
labels->resize(features_count);
|
||||
|
||||
for (size_t i = 0, index = 0; (i != shard_count) && (index < features_count);
|
||||
++i) {
|
||||
size_t next_index = index + fregment_count;
|
||||
if (next_index > features_count) {
|
||||
next_index = features_count;
|
||||
}
|
||||
|
||||
// Process in work thread
|
||||
task_group->submit(ailego::Closure::New(
|
||||
this, &KmeansCluster::update_labels_thread, index, next_index, labels));
|
||||
|
||||
// Next index
|
||||
index = next_index;
|
||||
}
|
||||
task_group->wait_finish();
|
||||
}
|
||||
|
||||
void KmeansCluster::split_clusters_thread(size_t index_begin, size_t index_end,
|
||||
const IndexThreads *threads) {
|
||||
size_t feature_size = features_->element_size();
|
||||
size_t thread_offset = threads->indexof_this() * seeker_->original()->count();
|
||||
|
||||
for (size_t i = index_begin; i != index_end; ++i) {
|
||||
const void *feat = features_->element(i);
|
||||
LinearSeeker::Document result(0, std::numeric_limits<float>::max());
|
||||
|
||||
// ignore error
|
||||
seeker_->seek(feat, meta_.element_size(), &result);
|
||||
|
||||
size_t sel_column = thread_offset + result.index;
|
||||
shard_cluster_scores_[sel_column] += result.score;
|
||||
shard_cluster_means_->at(sel_column).plus(feat, feature_size);
|
||||
}
|
||||
}
|
||||
|
||||
void KmeansCluster::update_centroid_thread(size_t column,
|
||||
IndexCluster::CentroidList *out) {
|
||||
size_t cluster_count = out->size();
|
||||
double cluster_score = 0.0;
|
||||
|
||||
// Create Accumulator
|
||||
std::shared_ptr<VectorMean> accum = NewVectorMean(meta_);
|
||||
if (batch_) {
|
||||
cluster_score += batch_scores_[column];
|
||||
accum->merge(batch_means_->at(column));
|
||||
}
|
||||
|
||||
// Compute the score of centroid
|
||||
for (size_t i = column; i < shard_cluster_scores_.size();
|
||||
i += cluster_count) {
|
||||
cluster_score += shard_cluster_scores_[i];
|
||||
accum->merge(shard_cluster_means_->at(i));
|
||||
}
|
||||
|
||||
// Update centroid
|
||||
IndexCluster::Centroid *centroid = &(out->at(column));
|
||||
centroid->set_score(cluster_score);
|
||||
centroid->set_follows(accum->count());
|
||||
accum->mean(centroid->mutable_buffer());
|
||||
}
|
||||
|
||||
void KmeansCluster::update_cluster_thread(size_t index_begin, size_t index_end,
|
||||
const IndexThreads *threads) {
|
||||
size_t thread_offset = threads->indexof_this() * seeker_->original()->count();
|
||||
|
||||
for (size_t i = index_begin; i != index_end; ++i) {
|
||||
const void *feat = features_->element(i);
|
||||
LinearSeeker::Document result(0, std::numeric_limits<float>::max());
|
||||
|
||||
// ignore error
|
||||
seeker_->seek(feat, meta_.element_size(), &result);
|
||||
|
||||
size_t sel_column = thread_offset + result.index;
|
||||
shard_cluster_features_[sel_column].emplace_back(feat);
|
||||
}
|
||||
}
|
||||
|
||||
void KmeansCluster::update_features_thread(size_t column,
|
||||
IndexCluster::CentroidList *out) {
|
||||
size_t cluster_count = out->size();
|
||||
size_t cluster_follows = 0u;
|
||||
|
||||
// Compute the follows of cluster
|
||||
for (size_t i = column; i < shard_cluster_features_.size();
|
||||
i += cluster_count) {
|
||||
cluster_follows += shard_cluster_features_[i].size();
|
||||
}
|
||||
|
||||
// Merge all features in cluster
|
||||
std::vector<const void *> &cluster_features =
|
||||
*(out->at(column).mutable_similars());
|
||||
cluster_features.resize(cluster_follows);
|
||||
|
||||
for (size_t i = column, j = 0; i < shard_cluster_features_.size();
|
||||
i += cluster_count) {
|
||||
const std::vector<const void *> &it = shard_cluster_features_[i];
|
||||
std::memcpy(&cluster_features[j], it.data(), it.size() * sizeof(void *));
|
||||
j += it.size();
|
||||
}
|
||||
}
|
||||
|
||||
void KmeansCluster::update_labels_thread(size_t index_begin, size_t index_end,
|
||||
std::vector<uint32_t> *labels) {
|
||||
for (size_t i = index_begin; i != index_end; ++i) {
|
||||
const void *feat = features_->element(i);
|
||||
LinearSeeker::Document result(0, std::numeric_limits<float>::max());
|
||||
|
||||
// ignore error
|
||||
seeker_->seek(feat, meta_.element_size(), &result);
|
||||
(*labels)[i] = static_cast<uint32_t>(result.index);
|
||||
}
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_CLUSTER_ALIAS(KmeansCluster, KmeansCluster, false);
|
||||
INDEX_FACTORY_REGISTER_CLUSTER_ALIAS(BatchKmeansCluster, KmeansCluster, true);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,45 @@
|
||||
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "linear_seeker.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
int LinearSeeker::seek(const void *query, size_t len, Document *out) {
|
||||
if (ailego_unlikely(!query || !out || meta_.element_size() != len)) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
float sel_score = std::numeric_limits<float>::max();
|
||||
uint32_t sel_column = 0;
|
||||
uint32_t total = static_cast<uint32_t>(features_->count());
|
||||
|
||||
for (uint32_t i = 0; i < total; ++i) {
|
||||
float score = 0.0f;
|
||||
|
||||
distance_func_(features_->element(i), query, meta_.dimension(), &score);
|
||||
if (score < sel_score) {
|
||||
sel_score = score;
|
||||
sel_column = i;
|
||||
}
|
||||
}
|
||||
|
||||
out->index = sel_column;
|
||||
out->score = sel_score;
|
||||
return 0;
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "seeker.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Linear Seeker
|
||||
*/
|
||||
class LinearSeeker : public Seeker {
|
||||
public:
|
||||
typedef std::shared_ptr<LinearSeeker> Pointer;
|
||||
|
||||
//! Constructor
|
||||
LinearSeeker(void) : meta_(), metric_(), features_() {}
|
||||
|
||||
//! Destructor
|
||||
~LinearSeeker(void) {}
|
||||
|
||||
//! Initialize Seeker
|
||||
int init(const IndexMeta &meta) override {
|
||||
meta_ = meta;
|
||||
|
||||
metric_ = IndexFactory::CreateMetric(meta_.metric_name());
|
||||
if (!metric_) {
|
||||
LOG_ERROR("Create Metric %s failed.", meta_.metric_name().c_str());
|
||||
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
int ret = metric_->init(meta_, meta_.metric_params());
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("IndexMetric init failed wit ret %d.", ret);
|
||||
|
||||
return ret;
|
||||
}
|
||||
distance_func_ = metric_->distance_matrix(1, 1);
|
||||
if (!distance_func_) {
|
||||
LOG_ERROR("DistanceMatrix function is nullptr.");
|
||||
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Seeker
|
||||
int cleanup(void) override {
|
||||
features_.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Reset Seeker
|
||||
int reset(void) override {
|
||||
features_.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Mount features
|
||||
int mount(IndexFeatures::Pointer feats) override {
|
||||
if (!feats) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!feats->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
features_ = std::move(feats);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Seek (TOP 1 Document)
|
||||
int seek(const void *query, size_t len, Document *out) override;
|
||||
|
||||
//! Retrieve the original features
|
||||
IndexFeatures::Pointer original(void) const override {
|
||||
return features_;
|
||||
}
|
||||
|
||||
private:
|
||||
IndexMeta meta_{};
|
||||
IndexMetric::Pointer metric_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
IndexMetric::MatrixDistance distance_func_{nullptr};
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_framework.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
class Seeker {
|
||||
public:
|
||||
struct Document {
|
||||
uint32_t index;
|
||||
float score;
|
||||
|
||||
//! Constructor
|
||||
Document(void) : index(0), score(0.0f) {}
|
||||
|
||||
//! Constructor
|
||||
Document(uint32_t i, float v) : index(i), score(v) {}
|
||||
|
||||
//! Constructor
|
||||
Document(const Document &rhs) : index(rhs.index), score(rhs.score) {}
|
||||
|
||||
//! Assignment
|
||||
Document &operator=(const Document &rhs) {
|
||||
index = rhs.index;
|
||||
score = rhs.score;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Less than
|
||||
bool operator<(const Document &rhs) const {
|
||||
return (this->score < rhs.score);
|
||||
}
|
||||
|
||||
//! Greater than
|
||||
bool operator>(const Document &rhs) const {
|
||||
return (this->score > rhs.score);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
//! Destructor
|
||||
virtual ~Seeker(void) {}
|
||||
|
||||
virtual int init(const IndexMeta &meta) = 0;
|
||||
|
||||
virtual int cleanup(void) = 0;
|
||||
|
||||
virtual int reset(void) = 0;
|
||||
|
||||
virtual int mount(IndexFeatures::Pointer feats) = 0;
|
||||
|
||||
virtual int seek(const void *query, size_t len, Document *out) = 0;
|
||||
|
||||
virtual IndexFeatures::Pointer original(void) const = 0;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,502 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include <ailego/container/params.h>
|
||||
#include "framework/index_cluster.h"
|
||||
#include "framework/index_error.h"
|
||||
#include "framework/index_factory.h"
|
||||
#include "cluster_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Stratified Cluster
|
||||
*/
|
||||
class StratifiedCluster : public IndexCluster {
|
||||
public:
|
||||
//! Constructor
|
||||
StratifiedCluster(void) {}
|
||||
|
||||
//! Destructor
|
||||
virtual ~StratifiedCluster(void) {}
|
||||
|
||||
//! Initialize Cluster
|
||||
virtual int init(const IndexMeta &meta, const ailego::Params ¶ms) {
|
||||
meta_ = meta;
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cleanup Cluster
|
||||
virtual int cleanup(void) {
|
||||
features_.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Reset Cluster
|
||||
virtual int reset(void) {
|
||||
features_.reset();
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Update Cluster
|
||||
virtual int update(const ailego::Params ¶ms) {
|
||||
this->update_params(params);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Suggest dividing to K clusters
|
||||
virtual void suggest(uint32_t k) {
|
||||
cluster_count_ = k;
|
||||
}
|
||||
|
||||
//! Mount features
|
||||
virtual int mount(IndexFeatures::Pointer feats) {
|
||||
if (!feats) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!feats->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
features_ = std::move(feats);
|
||||
return 0;
|
||||
}
|
||||
|
||||
//! Cluster
|
||||
virtual int cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Classify
|
||||
virtual int classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Label
|
||||
virtual int label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out);
|
||||
|
||||
protected:
|
||||
//! Test if it is valid
|
||||
bool is_valid(void) const {
|
||||
if (!features_ || !features_->count()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Update parameters
|
||||
void update_params(const ailego::Params ¶ms);
|
||||
|
||||
//! Check Centroids
|
||||
bool check_centroids(const IndexCluster::CentroidList ¢s);
|
||||
|
||||
//! Initialize Sub Clusters
|
||||
int init_sub_clusters(IndexCluster::Pointer *first,
|
||||
IndexCluster::Pointer *second);
|
||||
|
||||
//! Initialize First Cluster
|
||||
int init_first_cluster(IndexCluster::Pointer *first);
|
||||
|
||||
//! Initialize Second Cluster
|
||||
int init_second_cluster(IndexCluster::Pointer *second,
|
||||
IndexFeatures::Pointer features);
|
||||
|
||||
private:
|
||||
//! Members
|
||||
IndexMeta meta_{};
|
||||
IndexFeatures::Pointer features_{};
|
||||
uint32_t cluster_count_{0u};
|
||||
uint32_t thread_count_{0u};
|
||||
uint32_t first_cluster_count_{0u};
|
||||
uint32_t second_cluster_count_{0u};
|
||||
bool auto_tuning_{false};
|
||||
std::string first_cluster_class_{"OptKmeansCluster"};
|
||||
std::string second_cluster_class_{"OptKmeansCluster"};
|
||||
ailego::Params first_cluster_params_{};
|
||||
ailego::Params second_cluster_params_{};
|
||||
|
||||
// TODO: Maybe optimize later
|
||||
uint32_t second_threads_count_{10u}; // todo
|
||||
};
|
||||
|
||||
int StratifiedCluster::cluster(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
IndexCluster::Pointer first_cluster;
|
||||
int result = init_first_cluster(&first_cluster);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize the first cluster.");
|
||||
return result;
|
||||
}
|
||||
|
||||
if (first_cluster_count_) {
|
||||
first_cluster->suggest(first_cluster_count_);
|
||||
}
|
||||
|
||||
// The first clustering
|
||||
LOG_DEBUG("Clustering with first cluster: %s.", first_cluster_class_.c_str());
|
||||
result = first_cluster->cluster(threads, cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to cluster in first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = first_cluster->classify(threads, cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to classify in first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
// Cleanup for saving memory
|
||||
first_cluster.reset();
|
||||
|
||||
// Calculate the total cluster count
|
||||
uint32_t total_cluster_count = cents.size() * second_cluster_count_;
|
||||
if (cluster_count_) {
|
||||
total_cluster_count = cluster_count_;
|
||||
}
|
||||
|
||||
// Use thread_threads cluster instead
|
||||
uint32_t tail_threads = threads->count() % second_threads_count_;
|
||||
std::vector<std::shared_ptr<IndexThreads>> threads_cluster;
|
||||
|
||||
// TODO: reusing threads pool?
|
||||
// Incase the threads count less than second threads count
|
||||
if (threads->count() / second_threads_count_ == 0) {
|
||||
for (size_t threads_idx = 0; threads_idx < tail_threads; threads_idx++) {
|
||||
std::shared_ptr<IndexThreads> curr_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(1, false);
|
||||
threads_cluster.push_back(curr_threads);
|
||||
}
|
||||
} else {
|
||||
for (size_t threads_idx = 0; threads_idx < second_threads_count_;
|
||||
threads_idx++) {
|
||||
uint32_t curr_threads_count = threads->count() / second_threads_count_;
|
||||
if (threads_idx >= second_threads_count_ - tail_threads) {
|
||||
curr_threads_count++;
|
||||
}
|
||||
std::shared_ptr<IndexThreads> curr_threads =
|
||||
std::make_shared<SingleQueueIndexThreads>(curr_threads_count, false);
|
||||
threads_cluster.push_back(curr_threads);
|
||||
}
|
||||
}
|
||||
|
||||
auto task_group = threads->make_group();
|
||||
// The second clustering
|
||||
for (size_t i = 0; i < cents.size(); ++i) {
|
||||
if (cents[i].similars().empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
IndexThreads::Pointer &curr_threads =
|
||||
threads_cluster[i % (threads_cluster.size())];
|
||||
|
||||
task_group->submit(ailego::Closure::New(
|
||||
[this, &curr_threads, &total_cluster_count, ¢s](size_t index) {
|
||||
auto &it = cents[index];
|
||||
IndexCluster::Pointer second_cluster;
|
||||
std::shared_ptr<FlexibleIndexFeatures> features =
|
||||
std::make_shared<FlexibleIndexFeatures>(
|
||||
meta_, it.similars().data(), it.similars().size());
|
||||
|
||||
int ret = this->init_second_cluster(&second_cluster, features);
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to initialize the second cluster.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (auto_tuning_) {
|
||||
if (total_cluster_count) {
|
||||
double factor = static_cast<double>(it.similars().size()) /
|
||||
static_cast<double>(this->features_->count());
|
||||
second_cluster->suggest(
|
||||
std::max(static_cast<uint32_t>(
|
||||
std::floor(total_cluster_count * factor)),
|
||||
1u));
|
||||
}
|
||||
} else if (second_cluster_count_) {
|
||||
second_cluster->suggest(second_cluster_count_);
|
||||
}
|
||||
|
||||
LOG_DEBUG("Clustering with second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
ret = second_cluster->cluster(curr_threads, *(it.mutable_subitems()));
|
||||
if (ret != 0) {
|
||||
LOG_ERROR("Failed to cluster in second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
}
|
||||
},
|
||||
i));
|
||||
}
|
||||
task_group->wait_finish();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedCluster::classify(IndexThreads::Pointer threads,
|
||||
IndexCluster::CentroidList ¢s) {
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
IndexCluster::Pointer first_cluster, second_cluster;
|
||||
int result = init_sub_clusters(&first_cluster, &second_cluster);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize the subclusters.");
|
||||
return result;
|
||||
}
|
||||
|
||||
// The first classifying
|
||||
result = first_cluster->classify(threads, cents);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to classify in first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
// Cleanup for saving memory
|
||||
first_cluster.reset();
|
||||
|
||||
std::shared_ptr<FlexibleIndexFeatures> shell =
|
||||
std::make_shared<FlexibleIndexFeatures>(meta_);
|
||||
|
||||
// The second classifying
|
||||
for (IndexCluster::Centroid &it : cents) {
|
||||
const auto &feats = it.similars();
|
||||
|
||||
if (feats.empty()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
shell->mount(feats.data(), feats.size());
|
||||
result = second_cluster->mount(shell);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features for second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = second_cluster->classify(threads, *it.mutable_subitems());
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to classify in second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedCluster::label(IndexThreads::Pointer threads,
|
||||
const IndexCluster::CentroidList ¢s,
|
||||
std::vector<uint32_t> *out) {
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
if (cents.empty()) {
|
||||
LOG_ERROR("The input centroid's list is empty.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->check_centroids(cents)) {
|
||||
LOG_ERROR("The input centroid's list includes some invalid centroids.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
if (!this->is_valid()) {
|
||||
LOG_ERROR("The cluster is not ready.");
|
||||
return IndexError_NoReady;
|
||||
}
|
||||
|
||||
IndexCluster::Pointer first_cluster;
|
||||
int result = init_first_cluster(&first_cluster);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize the subclusters.");
|
||||
return result;
|
||||
}
|
||||
|
||||
result = first_cluster->label(threads, cents, out);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to label in first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void StratifiedCluster::update_params(const ailego::Params ¶ms) {
|
||||
params.get(GENERAL_THREAD_COUNT, &thread_count_);
|
||||
params.get(GENERAL_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(STRATIFIED_CLUSTER_COUNT, &cluster_count_);
|
||||
params.get(STRATIFIED_CLUSTER_FIRST_COUNT, &first_cluster_count_);
|
||||
params.get(STRATIFIED_CLUSTER_SECOND_COUNT, &second_cluster_count_);
|
||||
params.get(STRATIFIED_CLUSTER_FIRST_CLASS, &first_cluster_class_);
|
||||
params.get(STRATIFIED_CLUSTER_SECOND_CLASS, &second_cluster_class_);
|
||||
params.get(STRATIFIED_CLUSTER_FIRST_PARAMS, &first_cluster_params_);
|
||||
params.get(STRATIFIED_CLUSTER_SECOND_PARAMS, &second_cluster_params_);
|
||||
params.get(STRATIFIED_CLUSTER_AUTO_TUNING, &auto_tuning_);
|
||||
params.get(STRATIFIED_CLUSTER_SECOND_POOL_COUNT, &second_threads_count_);
|
||||
}
|
||||
|
||||
bool StratifiedCluster::check_centroids(
|
||||
const IndexCluster::CentroidList ¢s) {
|
||||
for (const auto &it : cents) {
|
||||
if (it.size() != meta_.element_size()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
int StratifiedCluster::init_sub_clusters(IndexCluster::Pointer *first,
|
||||
IndexCluster::Pointer *second) {
|
||||
IndexCluster::Pointer first_cluster =
|
||||
IndexFactory::CreateCluster(first_cluster_class_);
|
||||
|
||||
if (!first_cluster) {
|
||||
LOG_ERROR("Failed to create first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
IndexCluster::Pointer second_cluster =
|
||||
IndexFactory::CreateCluster(second_cluster_class_);
|
||||
|
||||
if (!second_cluster) {
|
||||
LOG_ERROR("Failed to create second cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int result = first_cluster->init(meta_, first_cluster_params_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = second_cluster->init(meta_, second_cluster_params_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = first_cluster->mount(features_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features for first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
*first = std::move(first_cluster);
|
||||
*second = std::move(second_cluster);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedCluster::init_first_cluster(IndexCluster::Pointer *first) {
|
||||
IndexCluster::Pointer first_cluster =
|
||||
IndexFactory::CreateCluster(first_cluster_class_);
|
||||
|
||||
if (!first_cluster) {
|
||||
LOG_ERROR("Failed to create first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int result = first_cluster->init(meta_, first_cluster_params_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = first_cluster->mount(features_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features for first cluster: %s.",
|
||||
first_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
*first = std::move(first_cluster);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedCluster::init_second_cluster(IndexCluster::Pointer *second,
|
||||
IndexFeatures::Pointer features) {
|
||||
IndexCluster::Pointer second_cluster =
|
||||
IndexFactory::CreateCluster(second_cluster_class_);
|
||||
|
||||
if (!second_cluster) {
|
||||
LOG_ERROR("Failed to create second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return IndexError_NoExist;
|
||||
}
|
||||
|
||||
int result = second_cluster->init(meta_, second_cluster_params_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
result = second_cluster->mount(features);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features for second cluster: %s.",
|
||||
second_cluster_class_.c_str());
|
||||
return result;
|
||||
}
|
||||
|
||||
*second = std::move(second_cluster);
|
||||
return 0;
|
||||
}
|
||||
|
||||
INDEX_FACTORY_REGISTER_CLUSTER(StratifiedCluster);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,341 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#include "stratified_cluster_trainer.h"
|
||||
#include <ailego/utility/string_helper.h>
|
||||
#include <ailego/utility/time_helper.h>
|
||||
#include "framework/index_error.h"
|
||||
#include "framework/index_factory.h"
|
||||
#include "framework/index_helper.h"
|
||||
#include "cluster_params.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
const std::string StratifiedClusterTrainer::SEP_TOKEN = "*";
|
||||
const std::string StratifiedClusterTrainer::DEFAULT_CLUSTER_CLASS =
|
||||
"OptKmeansCluster";
|
||||
|
||||
int StratifiedClusterTrainer::init_params(const ailego::Params ¶ms) {
|
||||
params.get(STRATIFIED_TRAINER_SAMPLE_COUNT, &sample_count_);
|
||||
params.get(STRATIFIED_TRAINER_SAMPLE_RATIO, &sample_ratio_);
|
||||
params.get(STRATIFIED_TRAINER_THREAD_COUNT, &thread_count_);
|
||||
cluster_auto_tuning_ = params.get_as_bool(STRATIFIED_TRAINER_AUTOAUNE);
|
||||
|
||||
std::string centroids_num =
|
||||
params.get_as_string(STRATIFIED_TRAINER_CLUSTER_COUNT);
|
||||
if (!centroids_num.empty()) {
|
||||
ailego::StringHelper::Split(centroids_num, SEP_TOKEN, ¢roid_num_vec_);
|
||||
for (size_t i = 0; i < centroid_num_vec_.size(); ++i) {
|
||||
if (centroid_num_vec_[i] == 0) {
|
||||
LOG_ERROR("Invalid centroid num %s", centroids_num.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
LOG_ERROR("No centroids_num configed.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
|
||||
size_t level_cnt = centroid_num_vec_.size();
|
||||
for (size_t i = 1; i <= level_cnt; ++i) {
|
||||
std::string level_params_key =
|
||||
STRATIFIED_TRAINER_PARAMS_IN_LEVEL_PREFIX + std::to_string(i);
|
||||
ailego::Params level_params;
|
||||
params.get(level_params_key, &level_params);
|
||||
cluster_params_.push_back(level_params);
|
||||
}
|
||||
|
||||
std::string cluster_class(DEFAULT_CLUSTER_CLASS);
|
||||
params.get(STRATIFIED_TRAINER_CLASS_NAME, &cluster_class);
|
||||
ailego::StringHelper::Split(cluster_class, SEP_TOKEN, &cluster_class_);
|
||||
if (cluster_class_.size() == 1) {
|
||||
// repeat the cluster class to level_cnt
|
||||
for (size_t i = 1; i < level_cnt; ++i) {
|
||||
cluster_class_.push_back(cluster_class_[0]);
|
||||
}
|
||||
} else if (cluster_class_.size() != level_cnt) {
|
||||
LOG_ERROR("Cluster class should be equal to level count");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedClusterTrainer::init(const IndexMeta &index_meta,
|
||||
const ailego::Params ¶ms) {
|
||||
int err = init_params(params);
|
||||
if (err != 0) {
|
||||
LOG_ERROR("init params failed, errno:%d,%s", err, IndexError::What(err));
|
||||
return err;
|
||||
}
|
||||
|
||||
meta_ = index_meta;
|
||||
ailego::Params cluster_params;
|
||||
if (centroid_num_vec_.size() == 0) {
|
||||
LOG_ERROR("invalid centroid num");
|
||||
return IndexError_InvalidArgument;
|
||||
} else if (centroid_num_vec_.size() == 1) {
|
||||
// one level clustering
|
||||
class_name_ = cluster_class_[0];
|
||||
cluster_params = cluster_params_[0];
|
||||
suggest_centriod_cnt_ = centroid_num_vec_[0];
|
||||
} else if (centroid_num_vec_.size() == 2) {
|
||||
// cluster level > 1
|
||||
class_name_ = "StratifiedCluster";
|
||||
int level_cnt = centroid_num_vec_.size();
|
||||
cluster_params.set(STRATIFIED_CLUSTER_FIRST_CLASS,
|
||||
cluster_class_[level_cnt - 2]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_SECOND_CLASS,
|
||||
cluster_class_[level_cnt - 1]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_FIRST_COUNT,
|
||||
centroid_num_vec_[level_cnt - 2]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_SECOND_COUNT,
|
||||
centroid_num_vec_[level_cnt - 1]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_FIRST_PARAMS,
|
||||
cluster_params_[level_cnt - 2]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_SECOND_PARAMS,
|
||||
cluster_params_[level_cnt - 1]);
|
||||
cluster_params.set(STRATIFIED_CLUSTER_AUTO_TUNING, cluster_auto_tuning_);
|
||||
suggest_centriod_cnt_ =
|
||||
centroid_num_vec_[level_cnt - 1] * centroid_num_vec_[level_cnt - 2];
|
||||
} else {
|
||||
LOG_ERROR("Unsupported more than 2 level clustering.");
|
||||
return IndexError_Unsupported;
|
||||
}
|
||||
|
||||
cluster_ = IndexFactory::CreateCluster(class_name_);
|
||||
if (!cluster_) {
|
||||
LOG_ERROR("Failed to create cluster[%s]", class_name_.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
int result = cluster_->init(meta_, cluster_params);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to initialize of cluster[%s], error: %d, %s",
|
||||
class_name_.c_str(), result, IndexError::What(result));
|
||||
return result;
|
||||
}
|
||||
if (suggest_centriod_cnt_ > 0) {
|
||||
cluster_->suggest(suggest_centriod_cnt_);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedClusterTrainer::cleanup(void) {
|
||||
cluster_ = nullptr;
|
||||
centroids_.clear();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedClusterTrainer::train(IndexThreads::Pointer threads,
|
||||
IndexHolder::Pointer holder) {
|
||||
ailego::ElapsedTime timer;
|
||||
if (!holder) {
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
if (!holder->is_matched(meta_)) {
|
||||
return IndexError_Mismatch;
|
||||
}
|
||||
if (!threads) {
|
||||
threads = std::make_shared<SingleQueueIndexThreads>(thread_count_, false);
|
||||
if (!threads) {
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
}
|
||||
|
||||
size_t train_sample_count = std::max(
|
||||
sample_count_, static_cast<uint32_t>(sample_ratio_ * holder->count()));
|
||||
|
||||
IndexFeatures::Pointer features;
|
||||
if (train_sample_count > 0) {
|
||||
LOG_INFO(
|
||||
"Train sampling, SampleCount=%u, SampleRatio=%f, HolderCount=%lu, "
|
||||
"TrainCount=%lu",
|
||||
sample_count_, sample_ratio_, holder->count(), train_sample_count);
|
||||
|
||||
auto sampler = std::make_shared<SampleIndexFeatures<CompactIndexFeatures>>(
|
||||
meta_, train_sample_count);
|
||||
size_t pre_reserve = train_sample_count < holder->count()
|
||||
? train_sample_count
|
||||
: holder->count();
|
||||
sampler->reserve(pre_reserve);
|
||||
for (auto iter = holder->create_iterator(); iter && iter->is_valid();
|
||||
iter->next()) {
|
||||
sampler->emplace(iter->data());
|
||||
}
|
||||
features = sampler;
|
||||
stats_.set_trained_count(train_sample_count);
|
||||
} else {
|
||||
LOG_INFO(
|
||||
"Do no sampling, SampleCount=%u, SampleRatio=%f, "
|
||||
"HolderCount=%lu, TrainCount=%lu",
|
||||
sample_count_, sample_ratio_, holder->count(), holder->count());
|
||||
|
||||
auto no_sampler = std::make_shared<CompactIndexFeatures>(meta_);
|
||||
for (auto iter = holder->create_iterator(); iter && iter->is_valid();
|
||||
iter->next()) {
|
||||
no_sampler->emplace(iter->data());
|
||||
}
|
||||
|
||||
features = no_sampler;
|
||||
stats_.set_trained_count(holder->count());
|
||||
}
|
||||
stats_.set_discarded_count(0);
|
||||
|
||||
// Holder is not needed, cleanup it.
|
||||
holder.reset();
|
||||
|
||||
int result = cluster_->mount(features);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to mount features of cluster[%s], error: %d, %s",
|
||||
class_name_.c_str(), result, IndexError::What(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
centroids_.clear();
|
||||
result = cluster_->cluster(std::move(threads), centroids_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to cluster features of cluster[%s], error: %d, %s",
|
||||
class_name_.c_str(), result, IndexError::What(result));
|
||||
return result;
|
||||
}
|
||||
|
||||
// check build result
|
||||
std::vector<size_t> level_size;
|
||||
std::function<void(const IndexCluster::CentroidList &, size_t)>
|
||||
cal_centroid_cnt =
|
||||
[&cal_centroid_cnt, &level_size](
|
||||
const IndexCluster::CentroidList ¢s, size_t level) {
|
||||
if (level > level_size.size()) {
|
||||
level_size.resize(level);
|
||||
}
|
||||
level_size[level - 1] += cents.size();
|
||||
for (const auto &it : cents) {
|
||||
if (!it.subitems().empty()) {
|
||||
cal_centroid_cnt(it.subitems(), level + 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
cal_centroid_cnt(centroids_, 1);
|
||||
|
||||
size_t centroids_num = level_size[level_size.size() - 1];
|
||||
if (centroids_num > suggest_centriod_cnt_) {
|
||||
LOG_WARN(
|
||||
"Built centroid(%zd level) count[%zd] bigger than expected "
|
||||
"count[%d]",
|
||||
level_size.size(), centroids_num, suggest_centriod_cnt_);
|
||||
} else {
|
||||
LOG_INFO("Built centroid(%zd level) count[%zd], expected count[%d]",
|
||||
level_size.size(), centroids_num, suggest_centriod_cnt_);
|
||||
}
|
||||
|
||||
stats_.set_trained_costtime(timer.milli_seconds());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedClusterTrainer::load(IndexStorage::Pointer cntr) {
|
||||
if (!cntr) {
|
||||
LOG_ERROR("IndexStorage is nullptr.");
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
std::shared_ptr<MemoryIndexBundle> bundle =
|
||||
std::make_shared<MemoryIndexBundle>();
|
||||
if (!bundle) {
|
||||
LOG_ERROR("New MemoryInndexBundle failed.");
|
||||
return IndexError_NoMemory;
|
||||
}
|
||||
|
||||
auto results = cntr->get_all();
|
||||
for (auto &it : results) {
|
||||
IndexStorage::Segment::Pointer &seg = it.second;
|
||||
if (!seg) {
|
||||
LOG_ERROR("Get Segment %s failed.", it.first.c_str());
|
||||
return IndexError_InvalidArgument;
|
||||
}
|
||||
size_t data_size = seg->data_size();
|
||||
const void *data = nullptr;
|
||||
size_t actual_size = seg->read(0, &data, data_size);
|
||||
if (actual_size != data_size) {
|
||||
LOG_ERROR("Read data failed expect %zu, actual %zu.", data_size,
|
||||
actual_size);
|
||||
return IndexError_ReadData;
|
||||
}
|
||||
bundle->set(it.first, data, data_size);
|
||||
}
|
||||
|
||||
int result = IndexHelper::DeserializeFromStorage(cntr.get(), &meta_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to deserialize meta from container");
|
||||
return result;
|
||||
}
|
||||
|
||||
result = IndexCluster::Deserialize(meta_, std::move(bundle), ¢roids_);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to deserialize index: %d", result);
|
||||
return result;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int StratifiedClusterTrainer::dump(const IndexDumper::Pointer &dumper) {
|
||||
IndexBundle::Pointer bundle;
|
||||
int result = IndexCluster::Serialize(meta_, centroids_, &bundle);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("IndexCluster Serialize failed with ret %d.", result);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = IndexHelper::SerializeToDumper(meta_, dumper.get());
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Failed to serialize meta into dumper.");
|
||||
return result;
|
||||
}
|
||||
|
||||
for (const auto &it : bundle->all()) {
|
||||
size_t data_size = it.second.size();
|
||||
result = dumper->append(it.first, data_size, 0, 0);
|
||||
if (result != 0) {
|
||||
LOG_ERROR("Dumper append meta %s %zu failed.", it.first.c_str(),
|
||||
data_size);
|
||||
return IndexError_PackIndex;
|
||||
}
|
||||
size_t actual_size = dumper->write(it.second.buffer(), data_size);
|
||||
if (actual_size != data_size) {
|
||||
LOG_ERROR("Dumper segment %s expect %zu, actual %zu.", it.first.c_str(),
|
||||
data_size, actual_size);
|
||||
return IndexError_PackIndex;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const IndexMeta &StratifiedClusterTrainer::meta(void) const {
|
||||
return meta_;
|
||||
}
|
||||
|
||||
const IndexTrainer::Stats &StratifiedClusterTrainer::stats(void) const {
|
||||
return stats_;
|
||||
}
|
||||
|
||||
IndexBundle::Pointer StratifiedClusterTrainer::indexes(void) const {
|
||||
IndexBundle::Pointer bundle;
|
||||
IndexCluster::Serialize(meta_, centroids_, &bundle);
|
||||
return bundle;
|
||||
}
|
||||
|
||||
//! Register Cluster Trainer in Factory
|
||||
INDEX_FACTORY_REGISTER_TRAINER(StratifiedClusterTrainer);
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,84 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include "framework/index_cluster.h"
|
||||
#include "framework/index_trainer.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Cluster Trainer
|
||||
*/
|
||||
class StratifiedClusterTrainer : public IndexTrainer {
|
||||
public:
|
||||
typedef std::shared_ptr<StratifiedClusterTrainer> Pointer;
|
||||
|
||||
//! Constructor
|
||||
StratifiedClusterTrainer(void) {}
|
||||
|
||||
//! Destructor
|
||||
~StratifiedClusterTrainer(void) {}
|
||||
|
||||
protected:
|
||||
//! Initialize Trainer
|
||||
virtual int init(const IndexMeta &meta, const ailego::Params ¶ms);
|
||||
|
||||
//! Cleanup Trainer
|
||||
virtual int cleanup(void);
|
||||
|
||||
//! Train the data
|
||||
virtual int train(IndexThreads::Pointer threads, IndexHolder::Pointer holder);
|
||||
|
||||
//! Load index from file path or dir
|
||||
virtual int load(IndexStorage::Pointer cntr);
|
||||
|
||||
//! Dump index into file path or dir
|
||||
virtual int dump(const IndexDumper::Pointer &dumper);
|
||||
|
||||
//! Retrieve Index Meta
|
||||
virtual const IndexMeta &meta(void) const;
|
||||
|
||||
//! Retrieve statistics
|
||||
virtual const IndexTrainer::Stats &stats(void) const;
|
||||
|
||||
//! Retrieve the output indexes
|
||||
virtual IndexBundle::Pointer indexes(void) const;
|
||||
|
||||
private:
|
||||
int init_params(const ailego::Params ¶ms);
|
||||
|
||||
private:
|
||||
IndexMeta meta_{};
|
||||
uint32_t sample_count_{0u};
|
||||
float sample_ratio_{0.0};
|
||||
uint32_t thread_count_{0u};
|
||||
bool cluster_auto_tuning_{false};
|
||||
IndexCluster::Pointer cluster_{};
|
||||
IndexCluster::CentroidList centroids_{};
|
||||
|
||||
uint32_t suggest_centriod_cnt_{0u};
|
||||
std::string class_name_;
|
||||
std::vector<std::string> cluster_class_;
|
||||
std::vector<uint64_t> centroid_num_vec_;
|
||||
std::vector<ailego::Params> cluster_params_;
|
||||
IndexTrainer::Stats stats_{};
|
||||
|
||||
private:
|
||||
static const std::string SEP_TOKEN;
|
||||
static const std::string DEFAULT_CLUSTER_CLASS;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,791 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include <ailego/container/vector.h>
|
||||
#include <ailego/utility/float_helper.h>
|
||||
#include <ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace core {
|
||||
|
||||
/*! Vector Mean
|
||||
*/
|
||||
struct VectorMean {
|
||||
//! Destructor
|
||||
virtual ~VectorMean(void) {}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) = 0;
|
||||
|
||||
//! Plus a vector
|
||||
virtual bool plus(const void *vec, size_t len) = 0;
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual bool mean(void *out, size_t len) const = 0;
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const = 0;
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) = 0;
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const = 0;
|
||||
|
||||
//! Retrieve the dimension of vectors
|
||||
virtual size_t dimension(void) const = 0;
|
||||
};
|
||||
|
||||
/*! Vector Mean Array
|
||||
*/
|
||||
struct VectorMeanArray {
|
||||
//! Destructor
|
||||
virtual ~VectorMeanArray(void) {}
|
||||
|
||||
//! Operator []
|
||||
VectorMean &operator[](size_t i) {
|
||||
return this->at(i);
|
||||
}
|
||||
|
||||
//! Operator []
|
||||
const VectorMean &operator[](size_t i) const {
|
||||
return this->at(i);
|
||||
}
|
||||
|
||||
//! Resize accumulators
|
||||
virtual void resize(size_t cnt) = 0;
|
||||
|
||||
//! Clear accumulators
|
||||
virtual void clear(void) = 0;
|
||||
|
||||
//! Retrieve an accumulator
|
||||
virtual VectorMean &at(size_t i) = 0;
|
||||
|
||||
//! Retrieve an accumulator
|
||||
virtual const VectorMean &at(size_t i) const = 0;
|
||||
|
||||
//! Retrieve the count of accumulators
|
||||
virtual size_t count(void) const = 0;
|
||||
|
||||
//! Retrieve the dimension of accumulators
|
||||
virtual size_t dimension(void) const = 0;
|
||||
};
|
||||
|
||||
/*! General Vector Mean Array
|
||||
*/
|
||||
template <typename T, typename = typename std::is_base_of<VectorMean, T>::type>
|
||||
class GeneralVectorMeanArray : public VectorMeanArray {
|
||||
public:
|
||||
//! Constructor
|
||||
GeneralVectorMeanArray(size_t dim) : dimension_(dim), array_() {}
|
||||
|
||||
//! Constructor
|
||||
GeneralVectorMeanArray(const GeneralVectorMeanArray &rhs)
|
||||
: dimension_(rhs.dimension_), array_(rhs.array_) {}
|
||||
|
||||
//! Constructor
|
||||
GeneralVectorMeanArray(GeneralVectorMeanArray &&rhs)
|
||||
: dimension_(rhs.dimension_), array_(std::move(rhs.array_)) {}
|
||||
|
||||
//! Emplace an accumulator
|
||||
template <typename... TArgs>
|
||||
bool emplace(TArgs &&...args) {
|
||||
T accum(std::forward<TArgs>(args)...);
|
||||
if (accum.dimension() != dimension_) {
|
||||
return false;
|
||||
}
|
||||
array_.push_back(std::move(accum));
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Resize accumulators
|
||||
virtual void resize(size_t cnt) {
|
||||
if (array_.size() < cnt) {
|
||||
for (size_t i = array_.size(); i < cnt; ++i) {
|
||||
array_.emplace_back(dimension_);
|
||||
}
|
||||
} else {
|
||||
array_.resize(cnt);
|
||||
}
|
||||
}
|
||||
|
||||
//! Clear accumulators
|
||||
virtual void clear(void) {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
//! Retrieve an accumulator
|
||||
virtual VectorMean &at(size_t i) {
|
||||
return array_[i];
|
||||
}
|
||||
|
||||
//! Retrieve an accumulator
|
||||
virtual const VectorMean &at(size_t i) const {
|
||||
return array_[i];
|
||||
}
|
||||
|
||||
//! Retrieve the count of accumulators
|
||||
virtual size_t count(void) const {
|
||||
return array_.size();
|
||||
}
|
||||
|
||||
//! Retrieve the dimension of accumulators
|
||||
virtual size_t dimension(void) const {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Disable them
|
||||
GeneralVectorMeanArray(void) = delete;
|
||||
|
||||
//! Members
|
||||
size_t dimension_;
|
||||
std::vector<T> array_;
|
||||
};
|
||||
|
||||
/*! Numerical Vector Mean
|
||||
*/
|
||||
template <typename T,
|
||||
typename =
|
||||
typename std::enable_if<ailego::IsArithmetic<T>::value>::type>
|
||||
class NumericalVectorMean : public VectorMean {
|
||||
public:
|
||||
//! Constructor
|
||||
NumericalVectorMean(void) : count_(0), accums_() {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorMean(const NumericalVectorMean &rhs)
|
||||
: count_(rhs.count_), accums_(rhs.accums_) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorMean(NumericalVectorMean &&rhs)
|
||||
: count_(rhs.count_), accums_(std::move(rhs.accums_)) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorMean(size_t dim) : count_(0), accums_(dim) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorMean(const T *means, size_t dim, size_t cnt)
|
||||
: count_(cnt), accums_(dim) {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] = static_cast<double>(means[i]) * count_;
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
void reset(size_t dim) {
|
||||
count_ = 0u;
|
||||
accums_.clear();
|
||||
accums_.resize(dim, 0.0);
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) {
|
||||
this->reset(accums_.size());
|
||||
}
|
||||
|
||||
//! Plus a vector
|
||||
virtual bool plus(const void *vec, size_t len) {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += *(static_cast<const T *>(vec) + i);
|
||||
}
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual bool mean(void *out, size_t len) const {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
*(static_cast<T *>(out) + i) = FloatCast<T>(accums_[i] / count_);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const {
|
||||
ailego::NumericalVector<T> &vec =
|
||||
*static_cast<ailego::NumericalVector<T> *>(out);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
vec.resize(dim);
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
vec[i] = FloatCast<T>(accums_[i] / count_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) {
|
||||
const NumericalVectorMean<T> &src =
|
||||
dynamic_cast<const NumericalVectorMean<T> &>(rhs);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
if (dim != src.accums_.size()) {
|
||||
return false;
|
||||
}
|
||||
count_ += src.count_;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += src.accums_[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of accumulator
|
||||
virtual size_t dimension(void) const {
|
||||
return accums_.size();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<!std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(val);
|
||||
}
|
||||
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(std::round(val));
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t count_;
|
||||
std::vector<double> accums_;
|
||||
};
|
||||
|
||||
/*! Numerical Vector Harmonic Mean
|
||||
*/
|
||||
template <typename T,
|
||||
typename =
|
||||
typename std::enable_if<ailego::IsArithmetic<T>::value>::type>
|
||||
class NumericalVectorHarmonicMean : public VectorMean {
|
||||
public:
|
||||
//! Constructor
|
||||
NumericalVectorHarmonicMean(void) : count_(0), accums_() {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorHarmonicMean(const NumericalVectorHarmonicMean &rhs)
|
||||
: count_(rhs.count_), accums_(rhs.accums_) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorHarmonicMean(NumericalVectorHarmonicMean &&rhs)
|
||||
: count_(rhs.count_), accums_(std::move(rhs.accums_)) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorHarmonicMean(size_t dim) : count_(0), accums_(dim) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorHarmonicMean(const T *means, size_t dim, size_t cnt)
|
||||
: count_(cnt), accums_(dim) {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] = static_cast<double>(count_) / static_cast<double>(means[i]);
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
void reset(size_t dim) {
|
||||
count_ = 0u;
|
||||
accums_.clear();
|
||||
accums_.resize(dim, 0.0);
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) {
|
||||
this->reset(accums_.size());
|
||||
}
|
||||
|
||||
//! Plus a vector (harmonic)
|
||||
virtual bool plus(const void *vec, size_t len) {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += 1.0 / *(static_cast<const T *>(vec) + i);
|
||||
}
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors (harmonic)
|
||||
virtual bool mean(void *out, size_t len) const {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
*(static_cast<T *>(out) + i) = FloatCast<T>(count_ / accums_[i]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const {
|
||||
ailego::NumericalVector<T> &vec =
|
||||
*static_cast<ailego::NumericalVector<T> *>(out);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
vec.resize(dim);
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
vec[i] = FloatCast<T>(count_ / accums_[i]);
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) {
|
||||
const NumericalVectorHarmonicMean<T> &src =
|
||||
dynamic_cast<const NumericalVectorHarmonicMean<T> &>(rhs);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
if (dim != src.accums_.size()) {
|
||||
return false;
|
||||
}
|
||||
count_ += src.count_;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += src.accums_[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of accumulator
|
||||
virtual size_t dimension(void) const {
|
||||
return accums_.size();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<!std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(val);
|
||||
}
|
||||
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(std::round(val));
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t count_;
|
||||
std::vector<double> accums_;
|
||||
};
|
||||
|
||||
/*! Numerical Vector Geometric Mean
|
||||
*/
|
||||
template <typename T,
|
||||
typename =
|
||||
typename std::enable_if<ailego::IsArithmetic<T>::value>::type>
|
||||
class NumericalVectorGeometricMean : public VectorMean {
|
||||
public:
|
||||
//! Constructor
|
||||
NumericalVectorGeometricMean(void) : count_(0), accums_() {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorGeometricMean(const NumericalVectorGeometricMean &rhs)
|
||||
: count_(rhs.count_), accums_(rhs.accums_) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorGeometricMean(NumericalVectorGeometricMean &&rhs)
|
||||
: count_(rhs.count_), accums_(std::move(rhs.accums_)) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorGeometricMean(size_t dim) : count_(0), accums_(dim, 1.0) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorGeometricMean(const T *means, size_t dim, size_t cnt)
|
||||
: count_(cnt), accums_(dim, 1.0) {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] = std::pow(static_cast<double>(means[i]), count_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
void reset(size_t dim) {
|
||||
count_ = 0u;
|
||||
accums_.clear();
|
||||
accums_.resize(dim, 1.0);
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) {
|
||||
this->reset(accums_.size());
|
||||
}
|
||||
|
||||
//! Plus a vector (geometric)
|
||||
virtual bool plus(const void *vec, size_t len) {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] *= *(static_cast<const T *>(vec) + i);
|
||||
}
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors (geometric)
|
||||
virtual bool mean(void *out, size_t len) const {
|
||||
size_t dim = accums_.size();
|
||||
if (dim * sizeof(T) != len) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
*(static_cast<T *>(out) + i) =
|
||||
FloatCast<T>(std::pow(accums_[i], 1.0 / count_));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const {
|
||||
ailego::NumericalVector<T> &vec =
|
||||
*static_cast<ailego::NumericalVector<T> *>(out);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
vec.resize(dim);
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
vec[i] = FloatCast<T>(std::pow(accums_[i], 1.0 / count_));
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) {
|
||||
const NumericalVectorGeometricMean<T> &src =
|
||||
dynamic_cast<const NumericalVectorGeometricMean<T> &>(rhs);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
if (dim != src.accums_.size()) {
|
||||
return false;
|
||||
}
|
||||
count_ += src.count_;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] *= src.accums_[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of accumulator
|
||||
virtual size_t dimension(void) const {
|
||||
return accums_.size();
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<!std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(val);
|
||||
}
|
||||
|
||||
//! Convert float type to another type
|
||||
template <typename U>
|
||||
static auto FloatCast(const double &val) ->
|
||||
typename std::enable_if<std::is_integral<U>::value, U>::type {
|
||||
return static_cast<U>(std::round(val));
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t count_;
|
||||
std::vector<double> accums_;
|
||||
};
|
||||
|
||||
/*! Binary Vector Mean
|
||||
*/
|
||||
class BinaryVectorMean : public VectorMean {
|
||||
public:
|
||||
//! Constructor
|
||||
BinaryVectorMean(void) : count_(0), accums_() {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorMean(const BinaryVectorMean &rhs)
|
||||
: count_(rhs.count_), accums_(rhs.accums_) {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorMean(BinaryVectorMean &&rhs)
|
||||
: count_(rhs.count_), accums_(std::move(rhs.accums_)) {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorMean(size_t dim) : count_(0), accums_(((dim + 7) >> 3) << 3) {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorMean(const void *means, size_t dim, size_t cnt)
|
||||
: count_(cnt), accums_(((dim + 7) >> 3) << 3) {
|
||||
const uint8_t *bits = reinterpret_cast<const uint8_t *>(means);
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] = (count_ >> 1);
|
||||
|
||||
if (bits[i >> 3] & static_cast<uint8_t>(1 << (i & 0x7))) {
|
||||
accums_[i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
void reset(size_t dim) {
|
||||
count_ = 0u;
|
||||
accums_.clear();
|
||||
accums_.resize(dim);
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) {
|
||||
this->reset(accums_.size());
|
||||
}
|
||||
|
||||
//! Plus a vector
|
||||
virtual bool plus(const void *vec, size_t len) {
|
||||
size_t dim = accums_.size();
|
||||
if (dim != (len << 3)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint8_t *bits = reinterpret_cast<const uint8_t *>(vec);
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
if (bits[i >> 3] & static_cast<uint8_t>(1 << (i & 0x7))) {
|
||||
accums_[i] += 1;
|
||||
}
|
||||
}
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual bool mean(void *out, size_t len) const {
|
||||
size_t dim = accums_.size();
|
||||
if (dim != (len << 3)) {
|
||||
return false;
|
||||
}
|
||||
memset(out, 0, len);
|
||||
|
||||
uint8_t *bits = reinterpret_cast<uint8_t *>(out);
|
||||
size_t half_count = count_ >> 1;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
if (accums_[i] > half_count) {
|
||||
bits[i >> 3] |= static_cast<uint8_t>(1 << (i & 0x7));
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const {
|
||||
size_t dim = accums_.size();
|
||||
out->clear();
|
||||
out->resize((dim + 7) / 8);
|
||||
|
||||
uint8_t *bits =
|
||||
reinterpret_cast<uint8_t *>(const_cast<char *>(out->data()));
|
||||
size_t half_count = count_ >> 1;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
if (accums_[i] > half_count) {
|
||||
bits[i >> 3] |= static_cast<uint8_t>(1 << (i & 0x7));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) {
|
||||
const BinaryVectorMean &src = dynamic_cast<const BinaryVectorMean &>(rhs);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
if (dim != src.accums_.size()) {
|
||||
return false;
|
||||
}
|
||||
count_ += src.count_;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += src.accums_[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of accumulator
|
||||
virtual size_t dimension(void) const {
|
||||
return accums_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t count_;
|
||||
std::vector<size_t> accums_;
|
||||
};
|
||||
|
||||
/*! Numerical Vector Mean
|
||||
*/
|
||||
template <typename T,
|
||||
typename = typename std::enable_if<std::is_integral<T>::value>::type>
|
||||
class NibbleVectorMean : public VectorMean {
|
||||
public:
|
||||
//! Constructor
|
||||
NibbleVectorMean(void) : count_(0), accums_() {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorMean(const NibbleVectorMean &rhs)
|
||||
: count_(rhs.count_), accums_(rhs.accums_) {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorMean(NibbleVectorMean &&rhs)
|
||||
: count_(rhs.count_), accums_(std::move(rhs.accums_)) {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorMean(size_t dim) : count_(0), accums_(dim) {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorMean(const void *means, size_t dim, size_t cnt)
|
||||
: count_(cnt), accums_(dim) {
|
||||
const uint8_t *arr = reinterpret_cast<const uint8_t *>(means);
|
||||
for (size_t i = 0; i != dim; i += 2) {
|
||||
uint8_t val = arr[i >> 1];
|
||||
int lo = ((int8_t)(val << 4) >> 4);
|
||||
int hi = ((int8_t)(val) >> 4);
|
||||
accums_[i] = static_cast<double>(lo) * count_;
|
||||
accums_[i + 1] = static_cast<double>(hi) * count_;
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
void reset(size_t dim) {
|
||||
count_ = 0u;
|
||||
accums_.clear();
|
||||
accums_.resize(dim, 0.0);
|
||||
}
|
||||
|
||||
//! Reset accumulator
|
||||
virtual void reset(void) {
|
||||
this->reset(accums_.size());
|
||||
}
|
||||
|
||||
//! Plus a vector
|
||||
virtual bool plus(const void *vec, size_t len) {
|
||||
size_t dim = accums_.size();
|
||||
if (dim != (len << 1)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const uint8_t *arr = reinterpret_cast<const uint8_t *>(vec);
|
||||
for (size_t i = 0; i != dim; i += 2) {
|
||||
uint8_t val = arr[i >> 1];
|
||||
accums_[i] += ((int8_t)(val << 4) >> 4);
|
||||
accums_[i + 1] += ((int8_t)(val) >> 4);
|
||||
}
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual bool mean(void *out, size_t len) const {
|
||||
size_t dim = accums_.size();
|
||||
if (dim != (len << 1)) {
|
||||
return false;
|
||||
}
|
||||
memset(out, 0, len);
|
||||
|
||||
uint8_t *arr = reinterpret_cast<uint8_t *>(out);
|
||||
|
||||
for (size_t i = 0; i != dim; i += 2) {
|
||||
int lo = static_cast<int>(std::round(accums_[i] / count_));
|
||||
int hi = static_cast<int>(std::round(accums_[i + 1] / count_));
|
||||
arr[i >> 1] = (uint8_t)((hi << 4) & 0xf0) | (uint8_t)(lo & 0xf);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the mean of vectors
|
||||
virtual void mean(std::string *out) const {
|
||||
size_t dim = accums_.size();
|
||||
out->clear();
|
||||
out->resize(dim >> 1);
|
||||
|
||||
uint8_t *arr = reinterpret_cast<uint8_t *>(const_cast<char *>(out->data()));
|
||||
|
||||
for (size_t i = 0; i != dim; i += 2) {
|
||||
int lo = static_cast<int>(std::round(accums_[i] / count_));
|
||||
int hi = static_cast<int>(std::round(accums_[i + 1] / count_));
|
||||
arr[i >> 1] = (uint8_t)((hi << 4) & 0xf0) | (uint8_t)(lo & 0xf);
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another vector mean
|
||||
virtual bool merge(const VectorMean &rhs) {
|
||||
const NibbleVectorMean &src = dynamic_cast<const NibbleVectorMean &>(rhs);
|
||||
|
||||
size_t dim = accums_.size();
|
||||
if (dim != src.accums_.size()) {
|
||||
return false;
|
||||
}
|
||||
count_ += src.count_;
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
accums_[i] += src.accums_[i];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the count of vectors
|
||||
virtual size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve dimension of accumulator
|
||||
virtual size_t dimension(void) const {
|
||||
return accums_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t count_;
|
||||
std::vector<double> accums_;
|
||||
};
|
||||
|
||||
} // namespace core
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user