diff --git a/examples/c++/CMakeLists.txt b/examples/c++/CMakeLists.txt index d0dbf8b..d173157 100644 --- a/examples/c++/CMakeLists.txt +++ b/examples/c++/CMakeLists.txt @@ -43,7 +43,7 @@ set(zvec_ailego_deps ) set(zvec_core_deps - # empty + zvec_turbo ) set(zvec_db_deps diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c516187..00383c9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,6 +6,7 @@ git_version(ZVEC_VERSION ${CMAKE_CURRENT_SOURCE_DIR}) # Add repository cc_directory(ailego) +cc_directory(turbo) cc_directory(core) cc_directory(db) if(BUILD_PYTHON_BINDINGS) diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 7742db5..5f696c0 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -15,7 +15,7 @@ file(GLOB_RECURSE ALL_CORE_SRCS *.cc *.c *.h) cc_library( NAME zvec_core STATIC STRICT PACKED SRCS ${ALL_CORE_SRCS} - LIBS zvec_ailego sparsehash magic_enum + LIBS zvec_ailego zvec_turbo sparsehash magic_enum INCS . ${PROJECT_ROOT_DIR}/src/core VERSION "${GIT_SRCS_VER}" ) \ No newline at end of file diff --git a/src/core/metric/CMakeLists.txt b/src/core/metric/CMakeLists.txt index cbc1049..55dfc90 100644 --- a/src/core/metric/CMakeLists.txt +++ b/src/core/metric/CMakeLists.txt @@ -5,7 +5,7 @@ cc_library( NAME core_metric STATIC SHARED STRICT ALWAYS_LINK SRCS *.cc - LIBS zvec_ailego core_framework + LIBS zvec_ailego zvec_turbo core_framework INCS . ${PROJECT_ROOT_DIR}/src/core VERSION "${PROXIMA_ZVEC_VERSION}" ) diff --git a/src/core/metric/quantized_integer_metric.cc b/src/core/metric/quantized_integer_metric.cc index 2b4e757..f2c19cf 100644 --- a/src/core/metric/quantized_integer_metric.cc +++ b/src/core/metric/quantized_integer_metric.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include "metric_params.h" #include "quantized_integer_metric_batch.h" #include "quantized_integer_metric_matrix.h" @@ -95,6 +96,12 @@ class QuantizedIntegerMetric : public IndexMetric { switch (origin_metric_type_) { case MetricType::kSquaredEuclidean: if (meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_distance_func( + turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return DistanceMatrixCompute(m, n); } if (meta_.data_type() == IndexMeta::DataType::DT_INT4) { @@ -118,7 +125,6 @@ class QuantizedIntegerMetric : public IndexMetric { if (meta_.data_type() == IndexMeta::DataType::DT_INT4) { return DistanceMatrixCompute(m, n); } - // TODO: support MipsSquaredEuclidean other injection type break; case MetricType::kNormalizedCosine: @@ -131,6 +137,12 @@ class QuantizedIntegerMetric : public IndexMetric { break; case MetricType::kCosine: if (meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_distance_func( + turbo::MetricType::kCosine, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return DistanceMatrixCompute(m, n); } if (meta_.data_type() == IndexMeta::DataType::DT_INT4) { @@ -146,6 +158,12 @@ class QuantizedIntegerMetric : public IndexMetric { switch (origin_metric_type_) { case MetricType::kSquaredEuclidean: if (meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_batch_distance_func( + turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return reinterpret_cast( BaseDistanceBatchWithScoreUnquantized::ComputeBatch); @@ -180,7 +198,6 @@ class QuantizedIntegerMetric : public IndexMetric { BaseDistanceBatchWithScoreUnquantized< MipsSquaredEuclidean, uint8_t, 12, 2>::ComputeBatch); } - // TODO: support MipsSquaredEuclidean other injection type break; case MetricType::kNormalizedCosine: if (meta_.data_type() == IndexMeta::DataType::DT_INT8) { @@ -196,6 +213,12 @@ class QuantizedIntegerMetric : public IndexMetric { break; case MetricType::kCosine: if (meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_batch_distance_func( + turbo::MetricType::kCosine, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return reinterpret_cast( BaseDistanceBatchWithScoreUnquantized< CosineMinusInnerProduct, int8_t, 12, 2>::ComputeBatch); @@ -205,7 +228,6 @@ class QuantizedIntegerMetric : public IndexMetric { BaseDistanceBatchWithScoreUnquantized< CosineMinusInnerProduct, uint8_t, 12, 2>::ComputeBatch); } - break; } return nullptr; @@ -264,14 +286,25 @@ class QuantizedIntegerMetric : public IndexMetric { const override { if (origin_metric_type_ == MetricType::kCosine && meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_query_preprocess_func( + turbo::MetricType::kCosine, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return CosineMinusInnerProductDistanceBatchWithScoreUnquantized< int8_t, 1, 1>::GetQueryPreprocessFunc(); } else if (origin_metric_type_ == MetricType::kSquaredEuclidean && meta_.data_type() == IndexMeta::DataType::DT_INT8) { + auto turbo_ret = turbo::get_query_preprocess_func( + turbo::MetricType::kSquaredEuclidean, turbo::DataType::kInt8, + turbo::QuantizeType::kDefault); + if (turbo_ret) { + return turbo_ret; + } return SquaredEuclideanDistanceBatchWithScoreUnquantized< int8_t, 1, 1>::GetQueryPreprocessFunc(); } - return nullptr; } diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h new file mode 100644 index 0000000..6ecbfdd --- /dev/null +++ b/src/include/zvec/turbo/turbo.h @@ -0,0 +1,55 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include + +namespace zvec::turbo { + +using DistanceFunc = + std::function; +using BatchDistanceFunc = std::function; +using QueryPreprocessFunc = + zvec::ailego::DistanceBatch::DistanceBatchQueryPreprocessFunc; + +enum class MetricType { + kSquaredEuclidean, + kCosine, + kMipsSquaredEuclidean, + kUnknown, +}; + +enum class DataType { + kInt8, + kUnknown, +}; + +enum class QuantizeType { + kDefault, +}; + +DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, + QuantizeType quantize_type); + +BatchDistanceFunc get_batch_distance_func(MetricType metric_type, + DataType data_type, + QuantizeType quantize_type); + +QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type, + DataType data_type, + QuantizeType quantize_type); + +} // namespace zvec::turbo diff --git a/src/turbo/CMakeLists.txt b/src/turbo/CMakeLists.txt new file mode 100644 index 0000000..0aa834a --- /dev/null +++ b/src/turbo/CMakeLists.txt @@ -0,0 +1,36 @@ +include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) +include(${PROJECT_ROOT_DIR}/cmake/option.cmake) + +if(NOT ANDROID AND AUTO_DETECT_ARCH) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64") + setup_compiler_march_for_x86(TURBO_MARCH_FLAG_SSE TURBO_MARCH_FLAG_AVX2 TURBO_MARCH_FLAG_AVX512) + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") + # ARM64 architecture - no special march flags needed for now + # NEON implementations can be added here if needed + message(STATUS "turbo: ARM64 detected, skipping x86-specific optimizations") + endif() +endif() + +file(GLOB_RECURSE ALL_SRCS *.cc *.c *.h) + +# Set per-file compile flags for AVX512-VNNI sources. +# set_source_files_properties is directory-scoped, so it must be called in the +# same directory that adds the sources to a target (i.e. here, not in a +# subdirectory). +if(NOT ANDROID AND AUTO_DETECT_ARCH) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64") + file(GLOB_RECURSE AVX512_VNNI_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/avx512_vnni/*.cc) + set_source_files_properties( + ${AVX512_VNNI_SRCS} + PROPERTIES + COMPILE_FLAGS "${TURBO_MARCH_FLAG_AVX512}" + ) + endif() +endif() + +cc_library( + NAME zvec_turbo STATIC STRICT PACKED + SRCS ${ALL_SRCS} + LIBS zvec_ailego + INCS ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_ROOT_DIR}/src/include +) diff --git a/src/turbo/avx512_vnni/record_quantized_int8/common.h b/src/turbo/avx512_vnni/record_quantized_int8/common.h new file mode 100644 index 0000000..55fb589 --- /dev/null +++ b/src/turbo/avx512_vnni/record_quantized_int8/common.h @@ -0,0 +1,312 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Shared AVX512-VNNI inner product kernels for record_quantized_int8 distance +// implementations (cosine, l2, mips_l2, etc.). +// +// All functions are marked always_inline so that when this header is included +// from a per-file-march .cc translation unit, the compiler can fully inline +// and optimize them under the correct -march flag without any cross-TU call +// overhead. + +#pragma once + +#if defined(__AVX512VNNI__) +#include +#include +#include + +namespace zvec::turbo::avx512_vnni::internal { + +static inline int32_t HorizontalAdd_INT32_V256(__m256i v) { + __m256i x1 = _mm256_hadd_epi32(v, v); + __m256i x2 = _mm256_hadd_epi32(x1, x1); + __m128i x3 = _mm256_extractf128_si256(x2, 1); + __m128i x4 = _mm_add_epi32(_mm256_castsi256_si128(x2), x3); + return _mm_cvtsi128_si32(x4); +} + +#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast(m * q); + +// Compute the raw integer inner product of two int8 vectors of length `size`. +// The result is written to `*distance` as a float. +// Both `a` and `b` must point to int8_t arrays. +static __attribute__((always_inline)) void ip_int8_avx512_vnni( + const void *a, const void *b, size_t size, float *distance) { + const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); + const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); + + const int8_t *lhs = reinterpret_cast(a); + const int8_t *rhs = reinterpret_cast(b); + + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 6) << 6); + + float result = 0.0f; + + __m256i ymm_sum_0 = _mm256_setzero_si256(); + __m256i ymm_sum_1 = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); + + ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); + ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); + ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); + ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); + + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), + ONES_INT16_AVX), + ymm_sum_0); + ymm_sum_1 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), + ONES_INT16_AVX), + ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); + ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); + ymm_rhs = _mm256_abs_epi8(ymm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), + ONES_INT16_AVX), + ymm_sum_0); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), + ONES_INT16_SSE)), + ymm_sum_0); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); + + ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); + ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); + ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); + ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); + + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), + ONES_INT16_AVX), + ymm_sum_0); + ymm_sum_1 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), + ONES_INT16_AVX), + ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); + ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); + ymm_rhs = _mm256_abs_epi8(ymm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), + ONES_INT16_AVX), + ymm_sum_0); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), + ONES_INT16_SSE)), + ymm_sum_0); + lhs += 16; + rhs += 16; + } + } + result = static_cast( + HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); + + switch (last - lhs) { + case 15: + FMA_INT8_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + FMA_INT8_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + FMA_INT8_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + FMA_INT8_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + FMA_INT8_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + FMA_INT8_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + FMA_INT8_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + FMA_INT8_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + FMA_INT8_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_INT8_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_INT8_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_INT8_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_INT8_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_INT8_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_INT8_GENERAL(lhs[0], rhs[0], result) + } + *distance = result; +} + +#undef FMA_INT8_GENERAL + +// Shift the first `original_dim` bytes of `query` in-place from int8 to uint8 +// by adding 128 to each element. The metadata tail beyond `original_dim` is +// left untouched. This prepares the query for use with dpbusd (uint8 * int8). +static __attribute__((always_inline)) void shift_int8_to_uint8_avx512( + void *query, size_t original_dim) { + const int8_t *input = reinterpret_cast(query); + uint8_t *output = reinterpret_cast(query); + + // 128 represented as int8_t wraps to -128, but two's complement addition + // produces the correct uint8 result. + const __m512i offset = _mm512_set1_epi8(static_cast(128)); + + size_t i = 0; + for (; i + 64 <= original_dim; i += 64) { + __m512i data = + _mm512_loadu_si512(reinterpret_cast(input + i)); + __m512i shifted = _mm512_add_epi8(data, offset); + _mm512_storeu_si512(reinterpret_cast<__m512i *>(output + i), shifted); + } + for (; i < original_dim; ++i) { + output[i] = static_cast(static_cast(input[i]) + 128); + } +} + +// Compute raw integer inner products for a batch of int8 vectors against a +// single query. Uses AVX512-VNNI dpbusd instruction. +// `query` is treated as uint8 (preprocessed), `vectors[i]` as int8. +template +__attribute__((always_inline)) void ip_int8_batch_avx512_vnni_impl( + const void *query, const void *const *vectors, + const std::array &prefetch_ptrs, + size_t dimensionality, float *distances) { + __m512i accs[batch_size]; + for (size_t i = 0; i < batch_size; ++i) { + accs[i] = _mm512_setzero_si512(); + } + size_t dim = 0; + for (; dim + 64 <= dimensionality; dim += 64) { + __m512i q = _mm512_loadu_si512(reinterpret_cast( + reinterpret_cast(query) + dim)); + __m512i data_regs[batch_size]; + for (size_t i = 0; i < batch_size; ++i) { + data_regs[i] = _mm512_loadu_si512(reinterpret_cast( + reinterpret_cast(vectors[i]) + dim)); + } + for (size_t i = 0; i < batch_size; ++i) { + if (prefetch_ptrs[i]) { + _mm_prefetch( + reinterpret_cast( + reinterpret_cast(prefetch_ptrs[i]) + dim), + _MM_HINT_T0); + } + accs[i] = _mm512_dpbusd_epi32(accs[i], q, data_regs[i]); + } + } + std::array temp_results{}; + for (size_t i = 0; i < batch_size; ++i) { + temp_results[i] = _mm512_reduce_add_epi32(accs[i]); + } + for (; dim < dimensionality; ++dim) { + int q = static_cast(reinterpret_cast(query)[dim]); + for (size_t i = 0; i < batch_size; ++i) { + temp_results[i] += + q * + static_cast(reinterpret_cast(vectors[i])[dim]); + } + } + for (size_t i = 0; i < batch_size; ++i) { + distances[i] = static_cast(temp_results[i]); + } +} + +// Dispatch batched inner product over all `n` vectors with prefetching. +static __attribute__((always_inline)) void ip_int8_batch_avx512_vnni( + const void *const *vectors, const void *query, size_t n, size_t dim, + float *distances) { + static constexpr size_t batch_size = 2; + static constexpr size_t prefetch_step = 2; + size_t i = 0; + for (; i + batch_size <= n; i += batch_size) { + std::array prefetch_ptrs; + for (size_t j = 0; j < batch_size; ++j) { + if (i + j + batch_size * prefetch_step < n) { + prefetch_ptrs[j] = vectors[i + j + batch_size * prefetch_step]; + } else { + prefetch_ptrs[j] = nullptr; + } + } + ip_int8_batch_avx512_vnni_impl( + query, &vectors[i], prefetch_ptrs, dim, distances + i); + } + for (; i < n; i++) { + std::array prefetch_ptrs{nullptr}; + ip_int8_batch_avx512_vnni_impl<1>(query, &vectors[i], prefetch_ptrs, dim, + distances + i); + } +} + +} // namespace zvec::turbo::avx512_vnni::internal + +#endif // defined(__AVX512VNNI__) diff --git a/src/turbo/avx512_vnni/record_quantized_int8/cosine.cc b/src/turbo/avx512_vnni/record_quantized_int8/cosine.cc new file mode 100644 index 0000000..34843ba --- /dev/null +++ b/src/turbo/avx512_vnni/record_quantized_int8/cosine.cc @@ -0,0 +1,144 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is compiled with per-file -march=avx512vnni (set in CMakeLists.txt) +// so that all AVX512-VNNI intrinsics and the inlined inner product kernels from +// common.h are compiled with the correct target ISA. + +#include "avx512_vnni/record_quantized_int8/cosine.h" +#include "avx512_vnni/record_quantized_int8/common.h" +#if defined(__AVX512VNNI__) +#include +#endif + +// Tail layout for quantized INT8 cosine vectors: +// +// [ original_dim bytes: int8_t elements ] +// [ float scale_a ] (ma) +// [ float bias_a ] (mb) +// [ float sum_a ] (ms) +// [ float square_sum_a ] (ms2) +// [ int int8_sum ] (sum of raw int8 elements, used when query is +// preprocessed to uint8 via +128 shift) +// +// The query tail has the same layout (qa, qb, qs, qs2) without int8_sum. + +namespace zvec::turbo::avx512_vnni { + +void cosine_int8_distance(const void *a, const void *b, size_t dim, + float *distance) { +#if defined(__AVX512VNNI__) + // `dim` is the full encoded size; the original vector occupies dim-24 bytes. + const int original_dim = dim - 24; + if (original_dim <= 0) { + return; + } + + // Compute raw integer inner product over the original_dim bytes. + // Note: for the single-vector path there is no query preprocessing, so both + // sides are treated as int8_t (same as the non-preprocessed path in + // MinusInnerProductDistanceBatchWithScoreUnquantized). + internal::ip_int8_avx512_vnni(a, b, original_dim, distance); + + const float *a_tail = reinterpret_cast( + reinterpret_cast(a) + original_dim); + const float *b_tail = reinterpret_cast( + reinterpret_cast(b) + original_dim); + + float ma = a_tail[0]; + float mb = a_tail[1]; + float ms = a_tail[2]; + + float qa = b_tail[0]; + float qb = b_tail[1]; + float qs = b_tail[2]; + + // Dequantize and compute cosine distance: + // cosine_dist = -(ma * qa * ip + mb * qa * qs + qb * ma * ms + // + original_dim * qb * mb) + *distance = -(ma * qa * *distance + mb * qa * qs + qb * ma * ms + + static_cast(original_dim) * qb * mb); +#else + (void)a; + (void)b; + (void)dim; + (void)distance; +#endif +} + +void cosine_int8_batch_distance(const void *const *vectors, const void *query, + size_t n, size_t dim, float *distances) { +#if defined(__AVX512VNNI__) + // `dim` is the full encoded size; the original vector occupies dim-24 bytes. + const int original_dim = dim - 24; + if (original_dim <= 0) { + return; + } + + // Compute raw inner products for all vectors. The query has been preprocessed + // (int8 + 128 -> uint8) so dpbusd can be used via ip_int8_batch_avx512_vnni. + internal::ip_int8_batch_avx512_vnni(vectors, query, n, original_dim, + distances); + + const float *q_tail = reinterpret_cast( + reinterpret_cast(query) + original_dim); + float qa = q_tail[0]; + float qb = q_tail[1]; + float qs = q_tail[2]; + + for (int i = 0; i < n; ++i) { + const float *m_tail = reinterpret_cast( + reinterpret_cast(vectors[i]) + original_dim); + float ma = m_tail[0]; + float mb = m_tail[1]; + float ms = m_tail[2]; + // Correct for the +128 shift applied to the query during preprocessing: + // dpbusd computes sum(uint8_query[i] * int8_data[i]) + // = sum((int8_query[i] + 128) * int8_data[i]) + // = true_ip + 128 * sum(int8_data[i]) + // int8_sum is stored as the 5th int-sized field after the 4 floats. + int int8_sum = reinterpret_cast(m_tail)[4]; + float &result = distances[i]; + result -= 128.0f * static_cast(int8_sum); + + // Dequantize and compute cosine distance: + // cosine_dist = -(ma * qa * ip + mb * qa * qs + qb * ma * ms + // + original_dim * qb * mb) + result = -(ma * qa * result + mb * qa * qs + qb * ma * ms + + static_cast(original_dim) * qb * mb); + } +#else + (void)vectors; + (void)query; + (void)n; + (void)dim; + (void)distances; +#endif +} + +void cosine_int8_query_preprocess(void *query, size_t dim) { +#if defined(__AVX512VNNI__) + // The original vector occupies dim-24 bytes; only those bytes are shifted. + const int original_dim = static_cast(dim) - 24; + if (original_dim <= 0) { + return; + } + internal::shift_int8_to_uint8_avx512(query, original_dim); +#else + (void)query; + (void)dim; +#endif +} + +} // namespace zvec::turbo::avx512_vnni diff --git a/src/turbo/avx512_vnni/record_quantized_int8/cosine.h b/src/turbo/avx512_vnni/record_quantized_int8/cosine.h new file mode 100644 index 0000000..836af10 --- /dev/null +++ b/src/turbo/avx512_vnni/record_quantized_int8/cosine.h @@ -0,0 +1,39 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace zvec::turbo::avx512_vnni { + +// Compute cosine distance (negative inner product after normalization) between +// a single quantized INT8 vector pair. +// `dim` includes the original vector bytes plus a 24-byte metadata tail +// (3 floats: scale_a, bias_a, sum_a). +void cosine_int8_distance(const void *a, const void *b, size_t dim, + float *distance); + +// Batch version of cosine_int8_distance. +// The query must have been preprocessed by cosine_int8_query_preprocess +// (int8 -> uint8 via +128 shift) before calling this function. +void cosine_int8_batch_distance(const void *const *vectors, const void *query, + size_t n, size_t dim, float *distances); + +// Preprocess the query vector in-place (shift int8 -> uint8 by adding 128) +// so that the AVX512-VNNI dpbusd instruction can be used for inner product. +// `dim` includes the 24-byte metadata tail. +void cosine_int8_query_preprocess(void *query, size_t dim); + +} // namespace zvec::turbo::avx512_vnni diff --git a/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.cc b/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.cc new file mode 100644 index 0000000..4485023 --- /dev/null +++ b/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.cc @@ -0,0 +1,138 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is compiled with per-file -march=avx512vnni (set in CMakeLists.txt) +// so that all AVX512-VNNI intrinsics and the inlined inner product kernels from +// common.h are compiled with the correct target ISA. + +#include "avx512_vnni/record_quantized_int8/squared_euclidean.h" +#include "avx512_vnni/record_quantized_int8/common.h" +#if defined(__AVX512VNNI__) +#include +#endif + +// Tail layout for quantized INT8 squared Euclidean vectors: +// +// [ original_dim bytes: int8_t elements ] +// [ float scale_a ] (ma) +// [ float bias_a ] (mb) +// [ float sum_a ] (ms) +// [ float sum2_a ] (ms2) +// [ int int8_sum ] (sum of raw int8 elements, used for bias correction +// when the query has been shifted to uint8 via +128) +// +// Total tail size: 4 floats + 1 int = 20 bytes, so dim = original_dim + 20. + +namespace zvec::turbo::avx512_vnni { + +void squared_euclidean_int8_distance(const void *a, const void *b, size_t dim, + float *distance) { +#if defined(__AVX512VNNI__) + const int original_dim = dim - 20; + if (original_dim <= 0) { + return; + } + internal::ip_int8_avx512_vnni(a, b, original_dim, distance); + + const float *a_tail = reinterpret_cast( + reinterpret_cast(a) + original_dim); + const float *b_tail = reinterpret_cast( + reinterpret_cast(b) + original_dim); + + float ma = a_tail[0]; + float mb = a_tail[1]; + float ms = a_tail[2]; + float ms2 = a_tail[3]; + + float qa = b_tail[0]; + float qb = b_tail[1]; + float qs = b_tail[2]; + float qs2 = b_tail[3]; + + const float sum = qa * qs; + const float sum2 = qa * qa * qs2; + + *distance = ma * ma * ms2 + sum2 - 2 * ma * qa * *distance + + (mb - qb) * (mb - qb) * original_dim + + 2 * (mb - qb) * (ms * ma - sum); +#else + (void)a; + (void)b; + (void)dim; + (void)distance; +#endif +} + +void squared_euclidean_int8_batch_distance(const void *const *vectors, + const void *query, size_t n, + size_t dim, float *distances) { +#if defined(__AVX512VNNI__) + const int original_dim = dim - 20; + if (original_dim <= 0) { + return; + } + + internal::ip_int8_batch_avx512_vnni(vectors, query, n, original_dim, + distances); + const float *q_tail = reinterpret_cast( + reinterpret_cast(query) + original_dim); + float qa = q_tail[0]; + float qb = q_tail[1]; + float qs = q_tail[2]; + float qs2 = q_tail[3]; + + const float sum = qa * qs; + const float sum2 = qa * qa * qs2; + for (size_t i = 0; i < n; ++i) { + const float *m_tail = reinterpret_cast( + reinterpret_cast(vectors[i]) + original_dim); + float ma = m_tail[0]; + float mb = m_tail[1]; + float ms = m_tail[2]; + float ms2 = m_tail[3]; + // Correct for the +128 shift applied to the query during preprocessing: + // dpbusd computes sum(uint8_query[i] * int8_data[i]) + // = sum((int8_query[i] + 128) * int8_data[i]) + // = true_ip + 128 * sum(int8_data[i]) + // int8_sum is stored as the 5th int-sized field after the 4 floats. + int int8_sum = reinterpret_cast(m_tail)[4]; + float &result = distances[i]; + result -= 128.0f * static_cast(int8_sum); + result = ma * ma * ms2 + sum2 - 2 * ma * qa * result + + (mb - qb) * (mb - qb) * original_dim + + 2 * (mb - qb) * (ms * ma - sum); + } +#else + (void)vectors; + (void)query; + (void)n; + (void)dim; + (void)distances; +#endif +} + +void squared_euclidean_int8_query_preprocess(void *query, size_t dim) { +#if defined(__AVX512VNNI__) + const int original_dim = static_cast(dim) - 20; + if (original_dim <= 0) { + return; + } + internal::shift_int8_to_uint8_avx512(query, original_dim); +#else + (void)query; + (void)dim; +#endif +} + +} // namespace zvec::turbo::avx512_vnni diff --git a/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.h b/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.h new file mode 100644 index 0000000..c183091 --- /dev/null +++ b/src/turbo/avx512_vnni/record_quantized_int8/squared_euclidean.h @@ -0,0 +1,41 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace zvec::turbo::avx512_vnni { + +// Compute squared Euclidean distance between a single quantized INT8 +// vector pair. +// `dim` includes the original vector bytes plus a 20-byte metadata tail +// (4 floats: scale_a, bias_a, sum_a, sum2_a). +void squared_euclidean_int8_distance(const void *a, const void *b, size_t dim, + float *distance); + +// Batch version of squared_euclidean_int8_distance. +// The query must have been preprocessed by +// squared_euclidean_int8_query_preprocess (int8 -> uint8 via +128 shift) +// before calling this function. +void squared_euclidean_int8_batch_distance(const void *const *vectors, + const void *query, size_t n, + size_t dim, float *distances); + +// Preprocess the query vector in-place (shift int8 -> uint8 by adding 128) +// for the batch path. Only the original_dim bytes are shifted; the metadata +// tail is left intact. `dim` includes the 20-byte metadata tail. +void squared_euclidean_int8_query_preprocess(void *query, size_t dim); + +} // namespace zvec::turbo::avx512_vnni diff --git a/src/turbo/turbo.cc b/src/turbo/turbo.cc new file mode 100644 index 0000000..a731cfe --- /dev/null +++ b/src/turbo/turbo.cc @@ -0,0 +1,75 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include "avx512_vnni/record_quantized_int8/cosine.h" +#include "avx512_vnni/record_quantized_int8/squared_euclidean.h" + +namespace zvec::turbo { + +DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, + QuantizeType quantize_type) { + if (data_type == DataType::kInt8) { + if (quantize_type == QuantizeType::kDefault) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (metric_type == MetricType::kSquaredEuclidean) { + return avx512_vnni::squared_euclidean_int8_distance; + } + if (metric_type == MetricType::kCosine) { + return avx512_vnni::cosine_int8_distance; + } + } + } + } + return nullptr; +} + +BatchDistanceFunc get_batch_distance_func(MetricType metric_type, + DataType data_type, + QuantizeType quantize_type) { + if (data_type == DataType::kInt8) { + if (quantize_type == QuantizeType::kDefault) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (metric_type == MetricType::kSquaredEuclidean) { + return avx512_vnni::squared_euclidean_int8_batch_distance; + } + if (metric_type == MetricType::kCosine) { + return avx512_vnni::cosine_int8_batch_distance; + } + } + } + } + return nullptr; +} + +QueryPreprocessFunc get_query_preprocess_func(MetricType metric_type, + DataType data_type, + QuantizeType quantize_type) { + if (data_type == DataType::kInt8) { + if (quantize_type == QuantizeType::kDefault) { + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + if (metric_type == MetricType::kSquaredEuclidean) { + return avx512_vnni::squared_euclidean_int8_query_preprocess; + } + if (metric_type == MetricType::kCosine) { + return avx512_vnni::cosine_int8_query_preprocess; + } + } + } + } + return nullptr; +} + +} // namespace zvec::turbo