feat: Introduce turbo. (#243)
This commit is contained in:
@@ -43,7 +43,7 @@ set(zvec_ailego_deps
|
||||
)
|
||||
|
||||
set(zvec_core_deps
|
||||
# empty
|
||||
zvec_turbo
|
||||
)
|
||||
|
||||
set(zvec_db_deps
|
||||
|
||||
@@ -6,6 +6,7 @@ git_version(ZVEC_VERSION ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
# Add repository
|
||||
cc_directory(ailego)
|
||||
cc_directory(turbo)
|
||||
cc_directory(core)
|
||||
cc_directory(db)
|
||||
if(BUILD_PYTHON_BINDINGS)
|
||||
|
||||
@@ -15,7 +15,7 @@ file(GLOB_RECURSE ALL_CORE_SRCS *.cc *.c *.h)
|
||||
cc_library(
|
||||
NAME zvec_core STATIC STRICT PACKED
|
||||
SRCS ${ALL_CORE_SRCS}
|
||||
LIBS zvec_ailego sparsehash magic_enum
|
||||
LIBS zvec_ailego zvec_turbo sparsehash magic_enum
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core
|
||||
VERSION "${GIT_SRCS_VER}"
|
||||
)
|
||||
@@ -5,7 +5,7 @@ cc_library(
|
||||
NAME core_metric
|
||||
STATIC SHARED STRICT ALWAYS_LINK
|
||||
SRCS *.cc
|
||||
LIBS zvec_ailego core_framework
|
||||
LIBS zvec_ailego zvec_turbo core_framework
|
||||
INCS . ${PROJECT_ROOT_DIR}/src/core
|
||||
VERSION "${PROXIMA_ZVEC_VERSION}"
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <ailego/math_batch/distance_batch.h>
|
||||
#include <zvec/core/framework/index_error.h>
|
||||
#include <zvec/core/framework/index_factory.h>
|
||||
#include <zvec/turbo/turbo.h>
|
||||
#include "metric_params.h"
|
||||
#include "quantized_integer_metric_batch.h"
|
||||
#include "quantized_integer_metric_matrix.h"
|
||||
@@ -95,6 +96,12 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
switch (origin_metric_type_) {
|
||||
case MetricType::kSquaredEuclidean:
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_distance_func(
|
||||
turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return DistanceMatrixCompute<SquaredEuclidean, int8_t>(m, n);
|
||||
}
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT4) {
|
||||
@@ -118,7 +125,6 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT4) {
|
||||
return DistanceMatrixCompute<MipsSquaredEuclidean, uint8_t>(m, n);
|
||||
}
|
||||
// TODO: support MipsSquaredEuclidean other injection type
|
||||
break;
|
||||
|
||||
case MetricType::kNormalizedCosine:
|
||||
@@ -131,6 +137,12 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
break;
|
||||
case MetricType::kCosine:
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_distance_func(
|
||||
turbo::MetricType::kCosine, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return DistanceMatrixCompute<CosineMinusInnerProduct, int8_t>(m, n);
|
||||
}
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT4) {
|
||||
@@ -146,6 +158,12 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
switch (origin_metric_type_) {
|
||||
case MetricType::kSquaredEuclidean:
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_batch_distance_func(
|
||||
turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return reinterpret_cast<IndexMetric::MatrixBatchDistanceHandle>(
|
||||
BaseDistanceBatchWithScoreUnquantized<SquaredEuclidean, int8_t,
|
||||
12, 2>::ComputeBatch);
|
||||
@@ -180,7 +198,6 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
BaseDistanceBatchWithScoreUnquantized<
|
||||
MipsSquaredEuclidean, uint8_t, 12, 2>::ComputeBatch);
|
||||
}
|
||||
// TODO: support MipsSquaredEuclidean other injection type
|
||||
break;
|
||||
case MetricType::kNormalizedCosine:
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
@@ -196,6 +213,12 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
break;
|
||||
case MetricType::kCosine:
|
||||
if (meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_batch_distance_func(
|
||||
turbo::MetricType::kCosine, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return reinterpret_cast<IndexMetric::MatrixBatchDistanceHandle>(
|
||||
BaseDistanceBatchWithScoreUnquantized<
|
||||
CosineMinusInnerProduct, int8_t, 12, 2>::ComputeBatch);
|
||||
@@ -205,7 +228,6 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
BaseDistanceBatchWithScoreUnquantized<
|
||||
CosineMinusInnerProduct, uint8_t, 12, 2>::ComputeBatch);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
return nullptr;
|
||||
@@ -264,14 +286,25 @@ class QuantizedIntegerMetric : public IndexMetric {
|
||||
const override {
|
||||
if (origin_metric_type_ == MetricType::kCosine &&
|
||||
meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_query_preprocess_func(
|
||||
turbo::MetricType::kCosine, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return CosineMinusInnerProductDistanceBatchWithScoreUnquantized<
|
||||
int8_t, 1, 1>::GetQueryPreprocessFunc();
|
||||
} else if (origin_metric_type_ == MetricType::kSquaredEuclidean &&
|
||||
meta_.data_type() == IndexMeta::DataType::DT_INT8) {
|
||||
auto turbo_ret = turbo::get_query_preprocess_func(
|
||||
turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8,
|
||||
turbo::QuantizeType::kDefault);
|
||||
if (turbo_ret) {
|
||||
return turbo_ret;
|
||||
}
|
||||
return SquaredEuclideanDistanceBatchWithScoreUnquantized<
|
||||
int8_t, 1, 1>::GetQueryPreprocessFunc();
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <zvec/ailego/math_batch/utils.h>
|
||||
|
||||
namespace zvec::turbo {
|
||||
|
||||
using DistanceFunc =
|
||||
std::function<void(const void *m, const void *q, size_t dim, float *out)>;
|
||||
using BatchDistanceFunc = std::function<void(
|
||||
const void **m, const void *q, size_t num, size_t dim, float *out)>;
|
||||
using QueryPreprocessFunc =
|
||||
zvec::ailego::DistanceBatch::DistanceBatchQueryPreprocessFunc;
|
||||
|
||||
enum class MetricType {
|
||||
kSquaredEuclidean,
|
||||
kCosine,
|
||||
kMipsSquaredEuclidean,
|
||||
kUnknown,
|
||||
};
|
||||
|
||||
enum class DataType {
|
||||
kInt8,
|
||||
kUnknown,
|
||||
};
|
||||
|
||||
enum class QuantizeType {
|
||||
kDefault,
|
||||
};
|
||||
|
||||
DistanceFunc get_distance_func(MetricType metric_type, DataType data_type,
|
||||
QuantizeType quantize_type);
|
||||
|
||||
BatchDistanceFunc get_batch_distance_func(MetricType metric_type,
|
||||
DataType data_type,
|
||||
QuantizeType quantize_type);
|
||||
|
||||
QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type,
|
||||
DataType data_type,
|
||||
QuantizeType quantize_type);
|
||||
|
||||
} // namespace zvec::turbo
|
||||
@@ -0,0 +1,36 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
if(NOT ANDROID AND AUTO_DETECT_ARCH)
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64")
|
||||
setup_compiler_march_for_x86(TURBO_MARCH_FLAG_SSE TURBO_MARCH_FLAG_AVX2 TURBO_MARCH_FLAG_AVX512)
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
|
||||
# ARM64 architecture - no special march flags needed for now
|
||||
# NEON implementations can be added here if needed
|
||||
message(STATUS "turbo: ARM64 detected, skipping x86-specific optimizations")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
file(GLOB_RECURSE ALL_SRCS *.cc *.c *.h)
|
||||
|
||||
# Set per-file compile flags for AVX512-VNNI sources.
|
||||
# set_source_files_properties is directory-scoped, so it must be called in the
|
||||
# same directory that adds the sources to a target (i.e. here, not in a
|
||||
# subdirectory).
|
||||
if(NOT ANDROID AND AUTO_DETECT_ARCH)
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64")
|
||||
file(GLOB_RECURSE AVX512_VNNI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/avx512_vnni/*.cc)
|
||||
set_source_files_properties(
|
||||
${AVX512_VNNI_SRCS}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${TURBO_MARCH_FLAG_AVX512}"
|
||||
)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME zvec_turbo STATIC STRICT PACKED
|
||||
SRCS ${ALL_SRCS}
|
||||
LIBS zvec_ailego
|
||||
INCS ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_ROOT_DIR}/src/include
|
||||
)
|
||||
@@ -0,0 +1,312 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Shared AVX512-VNNI inner product kernels for record_quantized_int8 distance
|
||||
// implementations (cosine, l2, mips_l2, etc.).
|
||||
//
|
||||
// All functions are marked always_inline so that when this header is included
|
||||
// from a per-file-march .cc translation unit, the compiler can fully inline
|
||||
// and optimize them under the correct -march flag without any cross-TU call
|
||||
// overhead.
|
||||
|
||||
#pragma once
|
||||
|
||||
#if defined(__AVX512VNNI__)
|
||||
#include <immintrin.h>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
|
||||
namespace zvec::turbo::avx512_vnni::internal {
|
||||
|
||||
static inline int32_t HorizontalAdd_INT32_V256(__m256i v) {
|
||||
__m256i x1 = _mm256_hadd_epi32(v, v);
|
||||
__m256i x2 = _mm256_hadd_epi32(x1, x1);
|
||||
__m128i x3 = _mm256_extractf128_si256(x2, 1);
|
||||
__m128i x4 = _mm_add_epi32(_mm256_castsi256_si128(x2), x3);
|
||||
return _mm_cvtsi128_si32(x4);
|
||||
}
|
||||
|
||||
#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast<float>(m * q);
|
||||
|
||||
// Compute the raw integer inner product of two int8 vectors of length `size`.
|
||||
// The result is written to `*distance` as a float.
|
||||
// Both `a` and `b` must point to int8_t arrays.
|
||||
static __attribute__((always_inline)) void ip_int8_avx512_vnni(
|
||||
const void *a, const void *b, size_t size, float *distance) {
|
||||
const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
|
||||
const int8_t *lhs = reinterpret_cast<const int8_t *>(a);
|
||||
const int8_t *rhs = reinterpret_cast<const int8_t *>(b);
|
||||
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
float result = 0.0f;
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
*distance = result;
|
||||
}
|
||||
|
||||
#undef FMA_INT8_GENERAL
|
||||
|
||||
// Shift the first `original_dim` bytes of `query` in-place from int8 to uint8
|
||||
// by adding 128 to each element. The metadata tail beyond `original_dim` is
|
||||
// left untouched. This prepares the query for use with dpbusd (uint8 * int8).
|
||||
static __attribute__((always_inline)) void shift_int8_to_uint8_avx512(
|
||||
void *query, size_t original_dim) {
|
||||
const int8_t *input = reinterpret_cast<const int8_t *>(query);
|
||||
uint8_t *output = reinterpret_cast<uint8_t *>(query);
|
||||
|
||||
// 128 represented as int8_t wraps to -128, but two's complement addition
|
||||
// produces the correct uint8 result.
|
||||
const __m512i offset = _mm512_set1_epi8(static_cast<int8_t>(128));
|
||||
|
||||
size_t i = 0;
|
||||
for (; i + 64 <= original_dim; i += 64) {
|
||||
__m512i data =
|
||||
_mm512_loadu_si512(reinterpret_cast<const __m512i *>(input + i));
|
||||
__m512i shifted = _mm512_add_epi8(data, offset);
|
||||
_mm512_storeu_si512(reinterpret_cast<__m512i *>(output + i), shifted);
|
||||
}
|
||||
for (; i < original_dim; ++i) {
|
||||
output[i] = static_cast<uint8_t>(static_cast<int>(input[i]) + 128);
|
||||
}
|
||||
}
|
||||
|
||||
// Compute raw integer inner products for a batch of int8 vectors against a
|
||||
// single query. Uses AVX512-VNNI dpbusd instruction.
|
||||
// `query` is treated as uint8 (preprocessed), `vectors[i]` as int8.
|
||||
template <size_t batch_size>
|
||||
__attribute__((always_inline)) void ip_int8_batch_avx512_vnni_impl(
|
||||
const void *query, const void *const *vectors,
|
||||
const std::array<const void *, batch_size> &prefetch_ptrs,
|
||||
size_t dimensionality, float *distances) {
|
||||
__m512i accs[batch_size];
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
accs[i] = _mm512_setzero_si512();
|
||||
}
|
||||
size_t dim = 0;
|
||||
for (; dim + 64 <= dimensionality; dim += 64) {
|
||||
__m512i q = _mm512_loadu_si512(reinterpret_cast<const __m512i *>(
|
||||
reinterpret_cast<const int8_t *>(query) + dim));
|
||||
__m512i data_regs[batch_size];
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
data_regs[i] = _mm512_loadu_si512(reinterpret_cast<const __m512i *>(
|
||||
reinterpret_cast<const int8_t *>(vectors[i]) + dim));
|
||||
}
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
if (prefetch_ptrs[i]) {
|
||||
_mm_prefetch(
|
||||
reinterpret_cast<const char *>(
|
||||
reinterpret_cast<const int8_t *>(prefetch_ptrs[i]) + dim),
|
||||
_MM_HINT_T0);
|
||||
}
|
||||
accs[i] = _mm512_dpbusd_epi32(accs[i], q, data_regs[i]);
|
||||
}
|
||||
}
|
||||
std::array<int, batch_size> temp_results{};
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
temp_results[i] = _mm512_reduce_add_epi32(accs[i]);
|
||||
}
|
||||
for (; dim < dimensionality; ++dim) {
|
||||
int q = static_cast<int>(reinterpret_cast<const uint8_t *>(query)[dim]);
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
temp_results[i] +=
|
||||
q *
|
||||
static_cast<int>(reinterpret_cast<const int8_t *>(vectors[i])[dim]);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < batch_size; ++i) {
|
||||
distances[i] = static_cast<float>(temp_results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Dispatch batched inner product over all `n` vectors with prefetching.
|
||||
static __attribute__((always_inline)) void ip_int8_batch_avx512_vnni(
|
||||
const void *const *vectors, const void *query, size_t n, size_t dim,
|
||||
float *distances) {
|
||||
static constexpr size_t batch_size = 2;
|
||||
static constexpr size_t prefetch_step = 2;
|
||||
size_t i = 0;
|
||||
for (; i + batch_size <= n; i += batch_size) {
|
||||
std::array<const void *, batch_size> prefetch_ptrs;
|
||||
for (size_t j = 0; j < batch_size; ++j) {
|
||||
if (i + j + batch_size * prefetch_step < n) {
|
||||
prefetch_ptrs[j] = vectors[i + j + batch_size * prefetch_step];
|
||||
} else {
|
||||
prefetch_ptrs[j] = nullptr;
|
||||
}
|
||||
}
|
||||
ip_int8_batch_avx512_vnni_impl<batch_size>(
|
||||
query, &vectors[i], prefetch_ptrs, dim, distances + i);
|
||||
}
|
||||
for (; i < n; i++) {
|
||||
std::array<const void *, 1> prefetch_ptrs{nullptr};
|
||||
ip_int8_batch_avx512_vnni_impl<1>(query, &vectors[i], prefetch_ptrs, dim,
|
||||
distances + i);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace zvec::turbo::avx512_vnni::internal
|
||||
|
||||
#endif // defined(__AVX512VNNI__)
|
||||
@@ -0,0 +1,144 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This file is compiled with per-file -march=avx512vnni (set in CMakeLists.txt)
|
||||
// so that all AVX512-VNNI intrinsics and the inlined inner product kernels from
|
||||
// common.h are compiled with the correct target ISA.
|
||||
|
||||
#include "avx512_vnni/record_quantized_int8/cosine.h"
|
||||
#include "avx512_vnni/record_quantized_int8/common.h"
|
||||
#if defined(__AVX512VNNI__)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
// Tail layout for quantized INT8 cosine vectors:
|
||||
//
|
||||
// [ original_dim bytes: int8_t elements ]
|
||||
// [ float scale_a ] (ma)
|
||||
// [ float bias_a ] (mb)
|
||||
// [ float sum_a ] (ms)
|
||||
// [ float square_sum_a ] (ms2)
|
||||
// [ int int8_sum ] (sum of raw int8 elements, used when query is
|
||||
// preprocessed to uint8 via +128 shift)
|
||||
//
|
||||
// The query tail has the same layout (qa, qb, qs, qs2) without int8_sum.
|
||||
|
||||
namespace zvec::turbo::avx512_vnni {
|
||||
|
||||
void cosine_int8_distance(const void *a, const void *b, size_t dim,
|
||||
float *distance) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
// `dim` is the full encoded size; the original vector occupies dim-24 bytes.
|
||||
const int original_dim = dim - 24;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute raw integer inner product over the original_dim bytes.
|
||||
// Note: for the single-vector path there is no query preprocessing, so both
|
||||
// sides are treated as int8_t (same as the non-preprocessed path in
|
||||
// MinusInnerProductDistanceBatchWithScoreUnquantized<int8_t>).
|
||||
internal::ip_int8_avx512_vnni(a, b, original_dim, distance);
|
||||
|
||||
const float *a_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(a) + original_dim);
|
||||
const float *b_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(b) + original_dim);
|
||||
|
||||
float ma = a_tail[0];
|
||||
float mb = a_tail[1];
|
||||
float ms = a_tail[2];
|
||||
|
||||
float qa = b_tail[0];
|
||||
float qb = b_tail[1];
|
||||
float qs = b_tail[2];
|
||||
|
||||
// Dequantize and compute cosine distance:
|
||||
// cosine_dist = -(ma * qa * ip + mb * qa * qs + qb * ma * ms
|
||||
// + original_dim * qb * mb)
|
||||
*distance = -(ma * qa * *distance + mb * qa * qs + qb * ma * ms +
|
||||
static_cast<float>(original_dim) * qb * mb);
|
||||
#else
|
||||
(void)a;
|
||||
(void)b;
|
||||
(void)dim;
|
||||
(void)distance;
|
||||
#endif
|
||||
}
|
||||
|
||||
void cosine_int8_batch_distance(const void *const *vectors, const void *query,
|
||||
size_t n, size_t dim, float *distances) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
// `dim` is the full encoded size; the original vector occupies dim-24 bytes.
|
||||
const int original_dim = dim - 24;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute raw inner products for all vectors. The query has been preprocessed
|
||||
// (int8 + 128 -> uint8) so dpbusd can be used via ip_int8_batch_avx512_vnni.
|
||||
internal::ip_int8_batch_avx512_vnni(vectors, query, n, original_dim,
|
||||
distances);
|
||||
|
||||
const float *q_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(query) + original_dim);
|
||||
float qa = q_tail[0];
|
||||
float qb = q_tail[1];
|
||||
float qs = q_tail[2];
|
||||
|
||||
for (int i = 0; i < n; ++i) {
|
||||
const float *m_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(vectors[i]) + original_dim);
|
||||
float ma = m_tail[0];
|
||||
float mb = m_tail[1];
|
||||
float ms = m_tail[2];
|
||||
// Correct for the +128 shift applied to the query during preprocessing:
|
||||
// dpbusd computes sum(uint8_query[i] * int8_data[i])
|
||||
// = sum((int8_query[i] + 128) * int8_data[i])
|
||||
// = true_ip + 128 * sum(int8_data[i])
|
||||
// int8_sum is stored as the 5th int-sized field after the 4 floats.
|
||||
int int8_sum = reinterpret_cast<const int *>(m_tail)[4];
|
||||
float &result = distances[i];
|
||||
result -= 128.0f * static_cast<float>(int8_sum);
|
||||
|
||||
// Dequantize and compute cosine distance:
|
||||
// cosine_dist = -(ma * qa * ip + mb * qa * qs + qb * ma * ms
|
||||
// + original_dim * qb * mb)
|
||||
result = -(ma * qa * result + mb * qa * qs + qb * ma * ms +
|
||||
static_cast<float>(original_dim) * qb * mb);
|
||||
}
|
||||
#else
|
||||
(void)vectors;
|
||||
(void)query;
|
||||
(void)n;
|
||||
(void)dim;
|
||||
(void)distances;
|
||||
#endif
|
||||
}
|
||||
|
||||
void cosine_int8_query_preprocess(void *query, size_t dim) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
// The original vector occupies dim-24 bytes; only those bytes are shifted.
|
||||
const int original_dim = static_cast<int>(dim) - 24;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
internal::shift_int8_to_uint8_avx512(query, original_dim);
|
||||
#else
|
||||
(void)query;
|
||||
(void)dim;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace zvec::turbo::avx512_vnni
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace zvec::turbo::avx512_vnni {
|
||||
|
||||
// Compute cosine distance (negative inner product after normalization) between
|
||||
// a single quantized INT8 vector pair.
|
||||
// `dim` includes the original vector bytes plus a 24-byte metadata tail
|
||||
// (3 floats: scale_a, bias_a, sum_a).
|
||||
void cosine_int8_distance(const void *a, const void *b, size_t dim,
|
||||
float *distance);
|
||||
|
||||
// Batch version of cosine_int8_distance.
|
||||
// The query must have been preprocessed by cosine_int8_query_preprocess
|
||||
// (int8 -> uint8 via +128 shift) before calling this function.
|
||||
void cosine_int8_batch_distance(const void *const *vectors, const void *query,
|
||||
size_t n, size_t dim, float *distances);
|
||||
|
||||
// Preprocess the query vector in-place (shift int8 -> uint8 by adding 128)
|
||||
// so that the AVX512-VNNI dpbusd instruction can be used for inner product.
|
||||
// `dim` includes the 24-byte metadata tail.
|
||||
void cosine_int8_query_preprocess(void *query, size_t dim);
|
||||
|
||||
} // namespace zvec::turbo::avx512_vnni
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// This file is compiled with per-file -march=avx512vnni (set in CMakeLists.txt)
|
||||
// so that all AVX512-VNNI intrinsics and the inlined inner product kernels from
|
||||
// common.h are compiled with the correct target ISA.
|
||||
|
||||
#include "avx512_vnni/record_quantized_int8/squared_euclidean.h"
|
||||
#include "avx512_vnni/record_quantized_int8/common.h"
|
||||
#if defined(__AVX512VNNI__)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
// Tail layout for quantized INT8 squared Euclidean vectors:
|
||||
//
|
||||
// [ original_dim bytes: int8_t elements ]
|
||||
// [ float scale_a ] (ma)
|
||||
// [ float bias_a ] (mb)
|
||||
// [ float sum_a ] (ms)
|
||||
// [ float sum2_a ] (ms2)
|
||||
// [ int int8_sum ] (sum of raw int8 elements, used for bias correction
|
||||
// when the query has been shifted to uint8 via +128)
|
||||
//
|
||||
// Total tail size: 4 floats + 1 int = 20 bytes, so dim = original_dim + 20.
|
||||
|
||||
namespace zvec::turbo::avx512_vnni {
|
||||
|
||||
void squared_euclidean_int8_distance(const void *a, const void *b, size_t dim,
|
||||
float *distance) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
const int original_dim = dim - 20;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
internal::ip_int8_avx512_vnni(a, b, original_dim, distance);
|
||||
|
||||
const float *a_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(a) + original_dim);
|
||||
const float *b_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(b) + original_dim);
|
||||
|
||||
float ma = a_tail[0];
|
||||
float mb = a_tail[1];
|
||||
float ms = a_tail[2];
|
||||
float ms2 = a_tail[3];
|
||||
|
||||
float qa = b_tail[0];
|
||||
float qb = b_tail[1];
|
||||
float qs = b_tail[2];
|
||||
float qs2 = b_tail[3];
|
||||
|
||||
const float sum = qa * qs;
|
||||
const float sum2 = qa * qa * qs2;
|
||||
|
||||
*distance = ma * ma * ms2 + sum2 - 2 * ma * qa * *distance +
|
||||
(mb - qb) * (mb - qb) * original_dim +
|
||||
2 * (mb - qb) * (ms * ma - sum);
|
||||
#else
|
||||
(void)a;
|
||||
(void)b;
|
||||
(void)dim;
|
||||
(void)distance;
|
||||
#endif
|
||||
}
|
||||
|
||||
void squared_euclidean_int8_batch_distance(const void *const *vectors,
|
||||
const void *query, size_t n,
|
||||
size_t dim, float *distances) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
const int original_dim = dim - 20;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
internal::ip_int8_batch_avx512_vnni(vectors, query, n, original_dim,
|
||||
distances);
|
||||
const float *q_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(query) + original_dim);
|
||||
float qa = q_tail[0];
|
||||
float qb = q_tail[1];
|
||||
float qs = q_tail[2];
|
||||
float qs2 = q_tail[3];
|
||||
|
||||
const float sum = qa * qs;
|
||||
const float sum2 = qa * qa * qs2;
|
||||
for (size_t i = 0; i < n; ++i) {
|
||||
const float *m_tail = reinterpret_cast<const float *>(
|
||||
reinterpret_cast<const int8_t *>(vectors[i]) + original_dim);
|
||||
float ma = m_tail[0];
|
||||
float mb = m_tail[1];
|
||||
float ms = m_tail[2];
|
||||
float ms2 = m_tail[3];
|
||||
// Correct for the +128 shift applied to the query during preprocessing:
|
||||
// dpbusd computes sum(uint8_query[i] * int8_data[i])
|
||||
// = sum((int8_query[i] + 128) * int8_data[i])
|
||||
// = true_ip + 128 * sum(int8_data[i])
|
||||
// int8_sum is stored as the 5th int-sized field after the 4 floats.
|
||||
int int8_sum = reinterpret_cast<const int *>(m_tail)[4];
|
||||
float &result = distances[i];
|
||||
result -= 128.0f * static_cast<float>(int8_sum);
|
||||
result = ma * ma * ms2 + sum2 - 2 * ma * qa * result +
|
||||
(mb - qb) * (mb - qb) * original_dim +
|
||||
2 * (mb - qb) * (ms * ma - sum);
|
||||
}
|
||||
#else
|
||||
(void)vectors;
|
||||
(void)query;
|
||||
(void)n;
|
||||
(void)dim;
|
||||
(void)distances;
|
||||
#endif
|
||||
}
|
||||
|
||||
void squared_euclidean_int8_query_preprocess(void *query, size_t dim) {
|
||||
#if defined(__AVX512VNNI__)
|
||||
const int original_dim = static_cast<int>(dim) - 20;
|
||||
if (original_dim <= 0) {
|
||||
return;
|
||||
}
|
||||
internal::shift_int8_to_uint8_avx512(query, original_dim);
|
||||
#else
|
||||
(void)query;
|
||||
(void)dim;
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace zvec::turbo::avx512_vnni
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace zvec::turbo::avx512_vnni {
|
||||
|
||||
// Compute squared Euclidean distance between a single quantized INT8
|
||||
// vector pair.
|
||||
// `dim` includes the original vector bytes plus a 20-byte metadata tail
|
||||
// (4 floats: scale_a, bias_a, sum_a, sum2_a).
|
||||
void squared_euclidean_int8_distance(const void *a, const void *b, size_t dim,
|
||||
float *distance);
|
||||
|
||||
// Batch version of squared_euclidean_int8_distance.
|
||||
// The query must have been preprocessed by
|
||||
// squared_euclidean_int8_query_preprocess (int8 -> uint8 via +128 shift)
|
||||
// before calling this function.
|
||||
void squared_euclidean_int8_batch_distance(const void *const *vectors,
|
||||
const void *query, size_t n,
|
||||
size_t dim, float *distances);
|
||||
|
||||
// Preprocess the query vector in-place (shift int8 -> uint8 by adding 128)
|
||||
// for the batch path. Only the original_dim bytes are shifted; the metadata
|
||||
// tail is left intact. `dim` includes the 20-byte metadata tail.
|
||||
void squared_euclidean_int8_query_preprocess(void *query, size_t dim);
|
||||
|
||||
} // namespace zvec::turbo::avx512_vnni
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <zvec/turbo/turbo.h>
|
||||
#include "avx512_vnni/record_quantized_int8/cosine.h"
|
||||
#include "avx512_vnni/record_quantized_int8/squared_euclidean.h"
|
||||
|
||||
namespace zvec::turbo {
|
||||
|
||||
DistanceFunc get_distance_func(MetricType metric_type, DataType data_type,
|
||||
QuantizeType quantize_type) {
|
||||
if (data_type == DataType::kInt8) {
|
||||
if (quantize_type == QuantizeType::kDefault) {
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
if (metric_type == MetricType::kSquaredEuclidean) {
|
||||
return avx512_vnni::squared_euclidean_int8_distance;
|
||||
}
|
||||
if (metric_type == MetricType::kCosine) {
|
||||
return avx512_vnni::cosine_int8_distance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
BatchDistanceFunc get_batch_distance_func(MetricType metric_type,
|
||||
DataType data_type,
|
||||
QuantizeType quantize_type) {
|
||||
if (data_type == DataType::kInt8) {
|
||||
if (quantize_type == QuantizeType::kDefault) {
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
if (metric_type == MetricType::kSquaredEuclidean) {
|
||||
return avx512_vnni::squared_euclidean_int8_batch_distance;
|
||||
}
|
||||
if (metric_type == MetricType::kCosine) {
|
||||
return avx512_vnni::cosine_int8_batch_distance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type,
|
||||
DataType data_type,
|
||||
QuantizeType quantize_type) {
|
||||
if (data_type == DataType::kInt8) {
|
||||
if (quantize_type == QuantizeType::kDefault) {
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
if (metric_type == MetricType::kSquaredEuclidean) {
|
||||
return avx512_vnni::squared_euclidean_int8_query_preprocess;
|
||||
}
|
||||
if (metric_type == MetricType::kCosine) {
|
||||
return avx512_vnni::cosine_int8_query_preprocess;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} // namespace zvec::turbo
|
||||
Reference in New Issue
Block a user