From a7bfd6060d0f084285b8bd7ba67380f46f0996ec Mon Sep 17 00:00:00 2001 From: rayx Date: Wed, 11 Mar 2026 22:44:48 +0800 Subject: [PATCH] refactor/march based reorganization (#193) * add for march compatible test * fix: remove MxN * fix: add macro scope * fix for android ci * refactor: call euclidean via squared euclidean * refactor: add match utility * refactor: comment config out * fix: fix dimension remainder * fix: fix condition & config flags * fix: remove redundant macros * fix: fix condition error * fix: fix macros * fix: remove unnecessary macros * fix: fix remaining code with vector * fix: remove unnecessary codes --------- Co-authored-by: Zefeng Yin Co-authored-by: xufeihong.xfh --- .github/workflows/android_build.yml | 11 + cmake/option.cmake | 72 +- pyproject.toml | 1 + src/ailego/CMakeLists.txt | 85 +- .../math/distance_matrix_euclidean_utility.i | 253 +++ .../distance_matrix_inner_product_utility.i | 208 ++ .../math/distance_matrix_mips_utility.i | 160 ++ src/ailego/math/euclidean_distance_matrix.h | 1922 ---------------- .../math/euclidean_distance_matrix_fp16.cc | 615 ------ .../euclidean_distance_matrix_fp16_avx.cc | 38 + .../euclidean_distance_matrix_fp16_avx512.cc | 96 + ...euclidean_distance_matrix_fp16_dispatch.cc | 87 + .../euclidean_distance_matrix_fp16_neon.cc | 35 + .../euclidean_distance_matrix_fp16_sse.cc | 54 + .../math/euclidean_distance_matrix_fp32.cc | 930 -------- .../euclidean_distance_matrix_fp32_avx.cc | 94 + .../euclidean_distance_matrix_fp32_avx512.cc | 81 + ...euclidean_distance_matrix_fp32_dispatch.cc | 92 + .../euclidean_distance_matrix_fp32_neon.cc | 62 + .../euclidean_distance_matrix_fp32_sse.cc | 78 + .../math/euclidean_distance_matrix_int4.cc | 801 ------- .../euclidean_distance_matrix_int4_avx2.cc | 118 + ...euclidean_distance_matrix_int4_dispatch.cc | 60 + .../euclidean_distance_matrix_int4_sse.cc | 98 + .../math/euclidean_distance_matrix_int8.cc | 884 -------- .../euclidean_distance_matrix_int8_avx2.cc | 182 ++ ...euclidean_distance_matrix_int8_dispatch.cc | 59 + .../euclidean_distance_matrix_int8_sse.cc | 164 ++ src/ailego/math/hamming_distance_matrix.cc | 765 ------- src/ailego/math/hamming_distance_matrix.h | 965 -------- src/ailego/math/inner_product_matrix.h | 1920 ---------------- src/ailego/math/inner_product_matrix_fp16.cc | 1948 ----------------- .../math/inner_product_matrix_fp16_avx.cc | 706 ++++++ .../math/inner_product_matrix_fp16_avx512.cc | 766 +++++++ .../inner_product_matrix_fp16_dispatch.cc | 162 ++ .../math/inner_product_matrix_fp16_neon.cc | 42 + src/ailego/math/inner_product_matrix_fp32.cc | 1180 ---------- .../math/inner_product_matrix_fp32_avx.cc | 94 + .../math/inner_product_matrix_fp32_avx512.cc | 75 + .../inner_product_matrix_fp32_dispatch.cc | 97 + .../math/inner_product_matrix_fp32_neon.cc | 57 + .../math/inner_product_matrix_fp32_sse.cc | 351 +++ src/ailego/math/inner_product_matrix_int4.cc | 803 ------- .../math/inner_product_matrix_int4_avx2.cc | 123 ++ .../inner_product_matrix_int4_dispatch.cc | 62 + .../math/inner_product_matrix_int4_sse.cc | 101 + src/ailego/math/inner_product_matrix_int8.cc | 841 ------- .../math/inner_product_matrix_int8_avx2.cc | 189 ++ .../inner_product_matrix_int8_dispatch.cc | 60 + .../math/inner_product_matrix_int8_sse.cc | 157 ++ .../mips_euclidean_distance_matrix_fp16.cc | 409 ---- ...mips_euclidean_distance_matrix_fp16_avx.cc | 116 + ...s_euclidean_distance_matrix_fp16_avx512.cc | 134 ++ ...euclidean_distance_matrix_fp16_dispatch.cc | 96 + ...ips_euclidean_distance_matrix_fp16_neon.cc | 126 ++ .../mips_euclidean_distance_matrix_fp32.cc | 684 ------ ...mips_euclidean_distance_matrix_fp32_avx.cc | 114 + ...s_euclidean_distance_matrix_fp32_avx512.cc | 100 + ...euclidean_distance_matrix_fp32_dispatch.cc | 136 ++ ...ips_euclidean_distance_matrix_fp32_neon.cc | 105 + ...mips_euclidean_distance_matrix_fp32_sse.cc | 336 +++ .../mips_euclidean_distance_matrix_int4.cc | 358 --- ...ips_euclidean_distance_matrix_int4_avx2.cc | 140 ++ ...euclidean_distance_matrix_int4_dispatch.cc | 83 + ...mips_euclidean_distance_matrix_int4_sse.cc | 104 + .../mips_euclidean_distance_matrix_int8.cc | 358 --- ...ips_euclidean_distance_matrix_int8_avx2.cc | 159 ++ ...euclidean_distance_matrix_int8_dispatch.cc | 81 + ...mips_euclidean_distance_matrix_int8_sse.cc | 137 ++ src/ailego/math/norm1_matrix_fp16.cc | 10 +- src/ailego/math/norm1_matrix_fp32.cc | 27 +- src/ailego/math/norm2_matrix_fp16.cc | 16 +- src/ailego/math/norm2_matrix_fp32.cc | 35 +- src/ailego/math_batch/distance_batch.h | 2 - .../math_batch/inner_product_distance_batch.h | 156 +- .../inner_product_distance_batch_dispatch.cc | 228 ++ ...r_product_distance_batch_impl_fp16_avx2.cc | 110 + ...roduct_distance_batch_impl_fp16_avx512.cc} | 108 +- ..._product_distance_batch_impl_fp32_avx2.cc} | 32 +- ...r_product_distance_batch_impl_int8_avx2.cc | 102 + ...roduct_distance_batch_impl_int8_avx512.cc} | 84 +- 81 files changed, 7801 insertions(+), 15689 deletions(-) create mode 100644 src/ailego/math/distance_matrix_euclidean_utility.i create mode 100644 src/ailego/math/distance_matrix_inner_product_utility.i create mode 100644 src/ailego/math/distance_matrix_mips_utility.i delete mode 100644 src/ailego/math/euclidean_distance_matrix_fp16.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp16_avx.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp16_avx512.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp16_dispatch.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp16_neon.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp16_sse.cc delete mode 100644 src/ailego/math/euclidean_distance_matrix_fp32.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp32_avx.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp32_avx512.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp32_neon.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_fp32_sse.cc delete mode 100644 src/ailego/math/euclidean_distance_matrix_int4.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int4_avx2.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int4_dispatch.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int4_sse.cc delete mode 100644 src/ailego/math/euclidean_distance_matrix_int8.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int8_avx2.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int8_dispatch.cc create mode 100644 src/ailego/math/euclidean_distance_matrix_int8_sse.cc delete mode 100644 src/ailego/math/inner_product_matrix_fp16.cc create mode 100644 src/ailego/math/inner_product_matrix_fp16_avx.cc create mode 100644 src/ailego/math/inner_product_matrix_fp16_avx512.cc create mode 100644 src/ailego/math/inner_product_matrix_fp16_dispatch.cc create mode 100644 src/ailego/math/inner_product_matrix_fp16_neon.cc delete mode 100644 src/ailego/math/inner_product_matrix_fp32.cc create mode 100644 src/ailego/math/inner_product_matrix_fp32_avx.cc create mode 100644 src/ailego/math/inner_product_matrix_fp32_avx512.cc create mode 100644 src/ailego/math/inner_product_matrix_fp32_dispatch.cc create mode 100644 src/ailego/math/inner_product_matrix_fp32_neon.cc create mode 100644 src/ailego/math/inner_product_matrix_fp32_sse.cc delete mode 100644 src/ailego/math/inner_product_matrix_int4.cc create mode 100644 src/ailego/math/inner_product_matrix_int4_avx2.cc create mode 100644 src/ailego/math/inner_product_matrix_int4_dispatch.cc create mode 100644 src/ailego/math/inner_product_matrix_int4_sse.cc delete mode 100644 src/ailego/math/inner_product_matrix_int8.cc create mode 100644 src/ailego/math/inner_product_matrix_int8_avx2.cc create mode 100644 src/ailego/math/inner_product_matrix_int8_dispatch.cc create mode 100644 src/ailego/math/inner_product_matrix_int8_sse.cc delete mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp16.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp16_avx.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp16_avx512.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp16_dispatch.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc delete mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32_avx.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc delete mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int4.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int4_avx2.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int4_dispatch.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int4_sse.cc delete mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int8.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int8_avx2.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int8_dispatch.cc create mode 100644 src/ailego/math/mips_euclidean_distance_matrix_int8_sse.cc create mode 100644 src/ailego/math_batch/inner_product_distance_batch_dispatch.cc create mode 100644 src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx2.cc rename src/ailego/math_batch/{inner_product_distance_batch_impl_fp16.h => inner_product_distance_batch_impl_fp16_avx512.cc} (70%) rename src/ailego/math_batch/{inner_product_distance_batch_impl.h => inner_product_distance_batch_impl_fp32_avx2.cc} (85%) create mode 100644 src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx2.cc rename src/ailego/math_batch/{inner_product_distance_batch_impl_int8.h => inner_product_distance_batch_impl_int8_avx512.cc} (68%) diff --git a/.github/workflows/android_build.yml b/.github/workflows/android_build.yml index 19706d8..099d486 100644 --- a/.github/workflows/android_build.yml +++ b/.github/workflows/android_build.yml @@ -12,6 +12,13 @@ on: - '**.md' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +permissions: + contents: read + jobs: build-android: # sdkmanager and other Android tools are x86‑only; ARM runners fail with exit code 1 @@ -161,6 +168,10 @@ jobs: adb shell getprop ro.product.cpu.abi adb shell getprop ro.product.cpu.abilist + echo "=== CPU ISA / Instruction Set Support ===" + echo "--- /proc/cpuinfo flags ---" + adb shell 'cat /proc/cpuinfo | grep -E "^(Features|flags)"' + echo "Checking binary sizes:" ls -lah examples/c++/build-android-examples-${{ matrix.abi }}/ diff --git a/cmake/option.cmake b/cmake/option.cmake index 0138856..71e4578 100644 --- a/cmake/option.cmake +++ b/cmake/option.cmake @@ -13,7 +13,7 @@ option(ENABLE_SAPPHIRERAPIDS "Enable Intel Sapphire Rapids Server CPU microarchi option(ENABLE_EMERALDRAPIDS "Enable Intel Emerald Rapids Server CPU microarchitecture" OFF) option(ENABLE_GRANITERAPIDS "Enable Intel Granite Rapids Server CPU microarchitecture" OFF) -option(ENABLE_NATIVE "Enable native CPU microarchitecture" ON) +option(ENABLE_NATIVE "Enable native CPU microarchitecture" OFF) ## AMD Microarchitectures option(ENABLE_ZEN1 "Enable AMD Zen+ Family 17h CPU microarchitecture" OFF) @@ -74,41 +74,57 @@ macro(add_arch_flag FLAG VAR_NAME OPTION_NAME) endif() endmacro() -function(_detect_armv8_best) - set(_arm_flags - "armv8.6-a" "armv8.5-a" "armv8.4-a" "armv8.3-a" "armv8.2-a" "armv8.1-a" "armv8-a" "armv8" - ) - foreach(_ver IN LISTS _arm_flags) - check_c_compiler_flag("-march=${_ver}" _COMP_SUPP_${_ver}) - if(_COMP_SUPP_${_ver}) - _AppendFlags(CMAKE_C_FLAGS "-march=${_ver}") - _AppendFlags(CMAKE_CXX_FLAGS "-march=${_ver}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE) - return() - endif() - endforeach() - message(WARNING "No ARMv8 architecture flag supported by compiler.") +function(_setup_armv8_march) + set(_arch "armv8") + check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch}) + if(_COMP_SUPP_${_arch}) + _AppendFlags(CMAKE_C_FLAGS "-march=${_arch}") + _AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE) + return() + else() + message(WARNING "No ARMv8 march flag supported by compiler.") + endif() endfunction() -function(_detect_x86_best) +function(_setup_x86_march) + set(_arch "x86-64") + check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch}) + if(_COMP_SUPP_${_arch}) + _AppendFlags(CMAKE_C_FLAGS "-march=${_arch}") + _AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE) + return() + else() + message(WARNING "No known x86 march flag supported; falling back to generic.") + endif() +endfunction() + +function(setup_compiler_march_for_x86 VAR_NAME_SSE VAR_NAME_AVX2 VAR_NAME_AVX512) + #sse + set(${VAR_NAME_SSE} "-march=corei7" PARENT_SCOPE) + + #avx 2 + set(${VAR_NAME_AVX2} "-march=core-avx2" PARENT_SCOPE) + + #avx512 set(_x86_flags - "graniterapids" "emeraldrapids" "sapphirerapids" - "skylake-avx512" "skylake" - "broadwell" "haswell" "sandybridge" "nehalem" - "znver3" "znver2" "znver1" + "graniterapids" "emeraldrapids" "sapphirerapids" "skylake-avx512" ) foreach(_arch IN LISTS _x86_flags) check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch}) if(_COMP_SUPP_${_arch}) - _AppendFlags(CMAKE_C_FLAGS "-march=${_arch}") - _AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE) + set(${VAR_NAME_AVX512} "-march=${_arch}" PARENT_SCOPE) return() endif() endforeach() - message(WARNING "No known x86 microarchitecture flag supported; falling back to generic.") + + + set(${VAR_NAME_AVX512} "-march=core-avx2" PARENT_SCOPE) + message(WARNING "No known avx512 microarchitecture flag found. Set up as core-avx2") + endfunction() if(MSVC) @@ -206,9 +222,9 @@ else() # AUTO DETECT # Heuristic: detect host architecture and probe appropriate flags if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") - _detect_armv8_best() + _setup_armv8_march() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64") - _detect_x86_best() + _setup_x86_march() else() message(WARNING "Unknown host architecture: ${CMAKE_SYSTEM_PROCESSOR}; no -march= set.") endif() diff --git a/pyproject.toml b/pyproject.toml index 5e99edf..349a6c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,6 +124,7 @@ sdist.include = [ [tool.scikit-build.cmake.define] BUILD_TOOLS = "OFF" BUILD_PYTHON_BINDINGS = "ON" +#CMAKE_VERBOSE_MAKEFILE = "ON" # Setuptools config for test pypi [tool.setuptools_scm] diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index 5fcaaca..c9867c3 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -18,6 +18,89 @@ if(UNIX AND NOT APPLE) list(APPEND EXTRA_LIBS ${LIB_RT}) endif() +if(NOT ANDROID) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64") + setup_compiler_march_for_x86(MATH_MARCH_FLAG_SSE MATH_MARCH_FLAG_AVX2 MATH_MARCH_FLAG_AVX512) + message(STATUS "best compiler march, sse: " ${MATH_MARCH_FLAG_SSE} ", avx2: " ${MATH_MARCH_FLAG_AVX2} ", avx512: " ${MATH_MARCH_FLAG_AVX512}) + + file(GLOB_RECURSE MATH_FILES_SSE + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_sse.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_sse.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_sse.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_sse.c + ) + + file(GLOB_RECURSE MATH_FILES_AVX2 + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx2.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx2.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx2.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx2.c + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx.c + ) + + file(GLOB_RECURSE MATH_FILES_AVX512 + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx512.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx512.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx512.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx512.c + ) + + foreach(MATH_FILE ${MATH_FILES_SSE}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_SSE}" + ) + endforeach() + + foreach(MATH_FILE ${MATH_FILES_AVX2}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_AVX2}" + ) + endforeach() + + foreach(MATH_FILE ${MATH_FILES_AVX512}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_AVX512}" + ) + endforeach() + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") + # set(CMAKE_CXX_FLAGS "-march=armv8-a") + # set(CMAKE_C_FLAGS "-march=armv8-a") + set(MATH_MARCH_FLAG_NEON "-march=armv8-a") + + file(GLOB_RECURSE MATH_FILES_NEON + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc + ${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c + ) + + foreach(MATH_FILE ${MATH_FILES_NEON}) + set_source_files_properties( + ${MATH_FILE} + PROPERTIES + COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}" + ) + endforeach() + endif() +endif() + cc_library( NAME zvec_ailego STATIC STRICT PACKED SRCS ${ALL_SRCS} @@ -25,4 +108,4 @@ cc_library( Arrow::arrow_static Arrow::parquet_static VERSION "${GIT_SRCS_VER}" -) \ No newline at end of file +) diff --git a/src/ailego/math/distance_matrix_euclidean_utility.i b/src/ailego/math/distance_matrix_euclidean_utility.i new file mode 100644 index 0000000..b0b8937 --- /dev/null +++ b/src/ailego/math/distance_matrix_euclidean_utility.i @@ -0,0 +1,253 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Calculate sum of squared difference (GENERAL) +#define SSD_FP32_GENERAL(m, q, sum) \ + { \ + float x = m - q; \ + sum += (x * x); \ + } + +//! Calculate sum of squared difference (SSE) +#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ + { \ + __m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \ + xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \ + } + +//! Calculate sum of squared difference (AVX) +#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ + { \ + __m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \ + ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \ + } + +//! Calculate sum of squared difference (NEON) +#define SSD_FP32_NEON(v_m, v_q, v_sum) \ + { \ + float32x4_t v_d = vsubq_f32(v_m, v_q); \ + v_sum = vfmaq_f32(v_sum, v_d, v_d); \ + } + +//! Calculate sum of squared difference (GENERAL) +#define SSD_FP16_GENERAL(m, q, sum) \ + { \ + float x = m - q; \ + sum += (x * x); \ + } + +//! Calculate sum of squared difference (NEON) +#define SSD_FP16_NEON(v_m, v_q, v_sum) \ + { \ + float16x8_t v_d = vsubq_f16(v_m, v_q); \ + v_sum = vfmaq_f16(v_sum, v_d, v_d); \ + } + +//! Calculate sum of squared difference (AVX512) +#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ + { \ + __m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \ + zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \ + } + +//! Calculate sum of squared difference (GENERAL) +#define SSD_INT4_GENERAL(m, q, sum) \ + sum += Int4SquaredDiffTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \ + Int4SquaredDiffTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)]; + + +#if defined(__SSE4_1__) +static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0xf0f0f0f0); +static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); +#endif // __SSE4_1__ + +//! Compute the square root of value (SSE) +#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v)) + +#if defined(__AVX2__) +static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0xf0f0f0f0); +static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); +#endif // __AVX2__ + +//! Calculate sum of squared difference (SSE) +#define SSD_INT4_SSE(xmm_m, xmm_q, xmm_sum) \ + { \ + __m128i xmm_lhs = \ + _mm_and_si128(_mm_slli_epi32((xmm_m), 4), MASK_INT4_SSE); \ + __m128i xmm_rhs = \ + _mm_and_si128(_mm_slli_epi32((xmm_q), 4), MASK_INT4_SSE); \ + xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \ + _mm_min_epi8(xmm_lhs, xmm_rhs)), \ + 4); \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \ + xmm_sum); \ + xmm_lhs = _mm_and_si128((xmm_m), MASK_INT4_SSE); \ + xmm_rhs = _mm_and_si128((xmm_q), MASK_INT4_SSE); \ + xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \ + _mm_min_epi8(xmm_lhs, xmm_rhs)), \ + 4); \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \ + xmm_sum); \ + } + +//! Compute the distance between matrix and query +#define SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ + { \ + __m128i xmm_lhs_0 = \ + _mm_and_si128(_mm_slli_epi32((xmm_lhs), 4), MASK_INT4_SSE); \ + __m128i xmm_rhs_0 = \ + _mm_and_si128(_mm_slli_epi32((xmm_rhs), 4), MASK_INT4_SSE); \ + __m128i xmm_lhs_1 = _mm_and_si128((xmm_lhs), MASK_INT4_SSE); \ + __m128i xmm_rhs_1 = _mm_and_si128((xmm_rhs), MASK_INT4_SSE); \ + xmm_lhs_0 = \ + _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), \ + _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)), \ + 4); \ + xmm_rhs_0 = \ + _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), \ + _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)), \ + 4); \ + xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs_0, xmm_lhs_0), \ + ONES_INT16_SSE); \ + xmm_rhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_rhs_0), \ + ONES_INT16_SSE); \ + xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_rhs_0), xmm_sum); \ + } + +//! Calculate sum of squared difference (AVX) +#define SSD_INT4_AVX(ymm_m, ymm_q, ymm_sum) \ + { \ + __m256i ymm_lhs = \ + _mm256_and_si256(_mm256_slli_epi32((ymm_m), 4), MASK_INT4_AVX); \ + __m256i ymm_rhs = \ + _mm256_and_si256(_mm256_slli_epi32((ymm_q), 4), MASK_INT4_AVX); \ + ymm_lhs = \ + _mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \ + _mm256_min_epi8(ymm_lhs, ymm_rhs)), \ + 4); \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \ + ONES_INT16_AVX), \ + ymm_sum); \ + ymm_lhs = _mm256_and_si256((ymm_m), MASK_INT4_AVX); \ + ymm_rhs = _mm256_and_si256((ymm_q), MASK_INT4_AVX); \ + ymm_lhs = \ + _mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \ + _mm256_min_epi8(ymm_lhs, ymm_rhs)), \ + 4); \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \ + ONES_INT16_AVX), \ + ymm_sum); \ + } + +//! Compute the distance between matrix and query +#define SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ + { \ + __m256i ymm_lhs_0 = \ + _mm256_and_si256(_mm256_slli_epi32((ymm_lhs), 4), MASK_INT4_AVX); \ + __m256i ymm_rhs_0 = \ + _mm256_and_si256(_mm256_slli_epi32((ymm_rhs), 4), MASK_INT4_AVX); \ + __m256i ymm_lhs_1 = _mm256_and_si256((ymm_lhs), MASK_INT4_AVX); \ + __m256i ymm_rhs_1 = _mm256_and_si256((ymm_rhs), MASK_INT4_AVX); \ + ymm_lhs_0 = _mm256_srli_epi32( \ + _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), \ + _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)), \ + 4); \ + ymm_rhs_0 = _mm256_srli_epi32( \ + _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), \ + _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)), \ + 4); \ + ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs_0, ymm_lhs_0), \ + ONES_INT16_AVX); \ + ymm_rhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_rhs_0), \ + ONES_INT16_AVX); \ + ymm_sum = \ + _mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_rhs_0), ymm_sum); \ + } + +//! Calculate sum of squared difference (GENERAL) +#define SSD_INT8_GENERAL(m, q, sum) \ + { \ + int32_t x = m - q; \ + sum += static_cast(x * x); \ + } + +//! Calculate sum of squared difference (SSE) +#define SSD_INT8_SSE(xmm_m, xmm_q, xmm_sum) \ + { \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_m), \ + _mm_sign_epi8(xmm_m, xmm_m)), \ + ONES_INT16_SSE), \ + xmm_sum); \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \ + _mm_sign_epi8(xmm_q, xmm_q)), \ + ONES_INT16_SSE), \ + xmm_sum); \ + xmm_sum = _mm_sub_epi32( \ + xmm_sum, \ + _mm_slli_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \ + _mm_sign_epi8(xmm_m, xmm_q)), \ + ONES_INT16_SSE), \ + 1)); \ + } + +//! Calculate sum of squared difference (AVX) +#define SSD_INT8_AVX(ymm_m, ymm_q, ymm_sum) \ + { \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_m), \ + _mm256_sign_epi8(ymm_m, ymm_m)), \ + ONES_INT16_AVX), \ + ymm_sum); \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ + _mm256_sign_epi8(ymm_q, ymm_q)), \ + ONES_INT16_AVX), \ + ymm_sum); \ + ymm_sum = _mm256_sub_epi32( \ + ymm_sum, _mm256_slli_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ + _mm256_sign_epi8(ymm_m, ymm_q)), \ + ONES_INT16_AVX), \ + 1)); \ + } + +//! Compute the square root of value (AVX) +#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v)) + +//! Compute the square root of value (AVX512) +#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v)) + +#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE +#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX + +#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512 +#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL + +#define ACCUM_FP16_STEP_NEON SSD_FP16_NEON +#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON + +#define ACCUM_INT4_STEP_SSE SSD_INT4_SSE +#define ACCUM_INT4_STEP_AVX SSD_INT4_AVX +#define ACCUM_INT8_STEP_SSE SSD_INT8_SSE +#define ACCUM_INT8_STEP_AVX SSD_INT8_AVX \ No newline at end of file diff --git a/src/ailego/math/distance_matrix_inner_product_utility.i b/src/ailego/math/distance_matrix_inner_product_utility.i new file mode 100644 index 0000000..3f28b15 --- /dev/null +++ b/src/ailego/math/distance_matrix_inner_product_utility.i @@ -0,0 +1,208 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(__SSE4_1__) +//! Four-bits Convert Table +static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = { + 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1}; + +#define NEGZEROS_FP32_SSE _mm_set1_ps(-0.0f) +#define MASK_INT4_SSE _mm_set1_epi32(0x0f0f0f0f) +#define ONES_INT16_SSE _mm_set1_epi32(0x00010001) +#define INT4_LOOKUP_SSE _mm_load_si128((const __m128i *)Int4ConvertTable) +#endif // __SSE4_1__ + +#if defined(__AVX__) +// #define NEGZEROS_FP32_AVX _mm256_set1_ps(-0.0f) +#define MASK_INT4_AVX _mm256_set1_epi32(0x0f0f0f0f) +#define ONES_INT16_AVX _mm256_set1_epi32(0x00010001) +#define INT4_LOOKUP_AVX _mm256_load_si256((const __m256i *)Int4ConvertTable) +#endif // __AVX__ + +#if defined(__AVX512F__) && !defined(__AVX512DQ__) +#define _mm512_xor_ps(a, b) \ + _mm512_castsi512_ps( \ + _mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b))) +#endif // __AVX512DQ__ + +//! Reverse sign of value (GENERAL) +#define NEGATE_FP32_GENERAL(v) -(v) + +//! Calculate Fused-Multiply-Add (SSE) +#define FMA_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ + xmm_sum = _mm_fmadd_ps(xmm_m, xmm_q, xmm_sum); + +//! Calculate Fused-Multiply-Add (AVX) +#define FMA_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ + ymm_sum = _mm256_fmadd_ps(ymm_m, ymm_q, ymm_sum); + +//! Calculate Fused-Multiply-Add (AVX512) +#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ + zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); + +//! Calculate Fused-Multiply-Add (AVX512FP16) +#define FMA_FP16_AVX512FP16(zmm_m, zmm_q, zmm_sum) \ + zmm_sum = _mm512_fmadd_ph(zmm_m, zmm_q, zmm_sum); + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_FP16_GENERAL(m, q, sum) sum += (m * q); + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_FP32_GENERAL(m, q, sum) sum += (m * q); + +//! Calculate Fused-Multiply-Add (NEON) +#define FMA_FP16_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f16(v_sum, v_m, v_q); + +//! Calculate Fused-Multiply-Add (NEON) +#define FMA_FP32_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_q); + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_INT4_GENERAL(m, q, sum) \ + sum += Int4MulTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \ + Int4MulTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)]; + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast(m * q); + +//! Calculate Fused-Multiply-Add (SSE) +#define FMA_INT8_SSE(xmm_m, xmm_q, xmm_sum) \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16( \ + _mm_maddubs_epi16(_mm_abs_epi8(xmm_q), _mm_sign_epi8(xmm_m, xmm_q)), \ + ONES_INT16_SSE), \ + xmm_sum); + +//! Calculate Fused-Multiply-Add (AVX) +#define FMA_INT8_AVX(ymm_m, ymm_q, ymm_sum) \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ + _mm256_sign_epi8(ymm_m, ymm_q)), \ + ONES_INT16_AVX), \ + ymm_sum); + +//! Calculate Fused-Multiply-Add (SSE) +#define FMA_INT4_SSE(xmm_m, xmm_q, xmm_sum) \ + { \ + __m128i xmm_lhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \ + _mm_and_si128((xmm_m), MASK_INT4_SSE)); \ + __m128i xmm_rhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \ + _mm_and_si128((xmm_q), MASK_INT4_SSE)); \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ + _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ + ONES_INT16_SSE), \ + xmm_sum); \ + xmm_lhs = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_m), 4), MASK_INT4_SSE)); \ + xmm_rhs = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_q), 4), MASK_INT4_SSE)); \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ + _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ + ONES_INT16_SSE), \ + xmm_sum); \ + } + +//! Calculate Fused-Multiply-Add (AVX) +#define FMA_INT4_AVX(ymm_m, ymm_q, ymm_sum) \ + { \ + __m256i ymm_lhs = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_m), MASK_INT4_AVX)); \ + __m256i ymm_rhs = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_q), MASK_INT4_AVX)); \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ + _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ + ONES_INT16_AVX), \ + ymm_sum); \ + ymm_lhs = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_m), 4), MASK_INT4_AVX)); \ + ymm_rhs = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_q), 4), MASK_INT4_AVX)); \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ + _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ + ONES_INT16_AVX), \ + ymm_sum); \ + } + +//! Compute the distance between matrix and query +#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ + { \ + __m128i xmm_lhs_0 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \ + __m128i xmm_rhs_0 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \ + __m128i xmm_lhs_1 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \ + __m128i xmm_rhs_1 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \ + xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); \ + xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); \ + xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); \ + xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); \ + xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), \ + ONES_INT16_SSE); \ + xmm_lhs_1 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), \ + ONES_INT16_SSE); \ + xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_lhs_1), xmm_sum); \ + } + +//! Compute the distance between matrix and query +#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ + { \ + __m256i ymm_lhs_0 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \ + __m256i ymm_rhs_0 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \ + __m256i ymm_lhs_1 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \ + __m256i ymm_rhs_1 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \ + ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); \ + ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); \ + ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); \ + ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); \ + ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), \ + ONES_INT16_AVX); \ + ymm_lhs_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), \ + ONES_INT16_AVX); \ + ymm_sum = \ + _mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_lhs_1), ymm_sum); \ + } + +#define ACCUM_FP16_STEP_GENERAL FMA_FP16_GENERAL +#define ACCUM_FP16_STEP_NEON FMA_FP16_NEON + +#define ACCUM_FP32_STEP_SSE FMA_FP32_SSE +#define ACCUM_FP32_STEP_AVX FMA_FP32_AVX +#define ACCUM_FP32_STEP_AVX512 FMA_FP32_AVX512 +#define ACCUM_FP32_STEP_NEON FMA_FP32_NEON + +#define ACCUM_INT4_STEP_SSE FMA_INT4_SSE +#define ACCUM_INT4_STEP_AVX FMA_INT4_AVX + +#define ACCUM_INT8_STEP_SSE FMA_INT8_SSE +#define ACCUM_INT8_STEP_AVX FMA_INT8_AVX diff --git a/src/ailego/math/distance_matrix_mips_utility.i b/src/ailego/math/distance_matrix_mips_utility.i new file mode 100644 index 0000000..871fdaa --- /dev/null +++ b/src/ailego/math/distance_matrix_mips_utility.i @@ -0,0 +1,160 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Calculate Fused-Multiply-Add (AVX512) +#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ + zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); + +#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \ + zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask); + +#define HorizontalAdd_FP16_NEON(v) \ + vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(v)), vcvt_high_f32_f16(v))) + +#define HorizontalAdd_FP32_V512_TO_V256(zmm) \ + _mm256_add_ps( \ + _mm512_castps512_ps256(zmm), \ + _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(zmm), 1))) + +//! Calculate Fused-Multiply-Add (AVX, FP16) +#define FMA_FP16_GENERAL(lhs, rhs, sum, norm1, norm2) \ + { \ + float v1 = lhs; \ + float v2 = rhs; \ + sum += v1 * v2; \ + norm1 += v1 * v1; \ + norm2 += v2 * v2; \ + } + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_FP32_GENERAL(lhs, rhs, sum, norm1, norm2) \ + { \ + sum += (lhs) * (rhs); \ + norm1 += (lhs) * (lhs); \ + norm2 += (rhs) * (rhs); \ + } + +#if defined(__SSE4_1__) +//! Four-bits Convert Table +static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = { + 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1, + 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1}; +#endif // __SSE4_1__ + +#if defined(__SSE4_1__) +static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f); +static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); +static const __m128i INT4_LOOKUP_SSE = + _mm_load_si128((const __m128i *)Int4ConvertTable); +#endif // __SSE4_1__ + +#if defined(__AVX2__) +static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f); +static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); +static const __m256i INT4_LOOKUP_AVX = + _mm256_load_si256((const __m256i *)Int4ConvertTable); +#endif // __AVX2__ + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_INT4_GENERAL(lhs, rhs, sum, norm1, norm2) \ + { \ + sum += Int4MulTable[(((lhs) << 4) & 0xf0) | (((rhs) >> 0) & 0xf)] + \ + Int4MulTable[(((lhs) >> 0) & 0xf0) | (((rhs) >> 4) & 0xf)]; \ + norm1 += static_cast( \ + ((int8_t)((lhs) << 4) >> 4) * ((int8_t)((lhs) << 4) >> 4) + \ + ((int8_t)((lhs) & 0xf0) >> 4) * ((int8_t)((lhs) & 0xf0) >> 4)); \ + norm2 += static_cast( \ + ((int8_t)((rhs) << 4) >> 4) * ((int8_t)((rhs) << 4) >> 4) + \ + ((int8_t)((rhs) & 0xf0) >> 4) * ((int8_t)((rhs) & 0xf0) >> 4)); \ + } + + +//! Compute the distance between matrix and query (SSE) +#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum_0, xmm_sum_norm1, \ + xmm_sum_norm2) \ + { \ + __m128i xmm_lhs_0 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \ + __m128i xmm_rhs_0 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \ + __m128i xmm_lhs_1 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \ + __m128i xmm_rhs_1 = _mm_shuffle_epi8( \ + INT4_LOOKUP_SSE, \ + _mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \ + FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); \ + FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); \ + FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); \ + FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum_0); \ + FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); \ + FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); \ + } + +//! Calculate Fused-Multiply-Add (GENERAL) +#define FMA_INT8_GENERAL(lhs, rhs, sum, norm1, norm2) \ + { \ + sum += static_cast(lhs * rhs); \ + norm1 += static_cast(lhs * lhs); \ + norm2 += static_cast(rhs * rhs); \ + } + +//! Calculate Fused-Multiply-Add (SSE) +#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ + xmm_sum = _mm_add_epi32( \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ + _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ + ONES_INT16_SSE), \ + xmm_sum) + +//! Calculate Fused-Multiply-Add (AVX) +#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ + ymm_sum = _mm256_add_epi32( \ + _mm256_madd_epi16( \ + _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ + _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ + ONES_INT16_AVX), \ + ymm_sum) + +#define FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum) \ + ymm_sum = _mm256_add_epi32( \ + _mm256_set_m128i( \ + _mm_setzero_si128(), \ + _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ + _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ + ONES_INT16_SSE)), \ + ymm_sum) + +//! Compute the distance between matrix and query (AVX) +#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, \ + ymm_sum_norm1, ymm_sum_norm2) \ + { \ + __m256i ymm_lhs_0 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \ + __m256i ymm_rhs_0 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \ + __m256i ymm_lhs_1 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \ + __m256i ymm_rhs_1 = _mm256_shuffle_epi8( \ + INT4_LOOKUP_AVX, \ + _mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \ + FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); \ + FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); \ + FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); \ + FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); \ + FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); \ + FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); \ + } + diff --git a/src/ailego/math/euclidean_distance_matrix.h b/src/ailego/math/euclidean_distance_matrix.h index 9155507..e8d5b4c 100644 --- a/src/ailego/math/euclidean_distance_matrix.h +++ b/src/ailego/math/euclidean_distance_matrix.h @@ -462,246 +462,6 @@ struct SquaredEuclideanDistanceMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -/*! Squared Euclidean Distance Matrix (FP32, M=2, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=2, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=4, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=4, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=4, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=8, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=8, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=8, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=8, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=16, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=16, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=16, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=16, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=16, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP32, M=32, N=32) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; #endif // __SSE__ || __ARM_NEON #if defined(__SSE__) || (defined(__ARM_NEON) && (defined(__aarch64__))) @@ -716,246 +476,6 @@ struct EuclideanDistanceMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -/*! Euclidean Distance Matrix (FP32, M=2, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=2, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=4, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=4, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=4, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=8, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=8, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=8, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=8, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=16, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=16, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=16, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=16, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=16, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP32, M=32, N=32) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; #endif // __SSE__ || __ARM_NEON && __aarch64__ #if (defined(__F16C__) && defined(__AVX__)) || \ @@ -983,488 +503,6 @@ struct EuclideanDistanceMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -#if !defined(__ARM_NEON) -/*! Squared Euclidean Distance Matrix (FP16, M=2, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=2, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=4, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=4, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=4, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=8, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=8, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=8, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=8, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=16, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=16, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=16, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=16, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=16, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (FP16, M=32, N=32) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=2, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=2, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=4, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=4, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=4, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=8, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=8, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=8, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=8, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=16, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=16, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=16, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=16, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=16, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (FP16, M=32, N=32) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // !__ARM_NEON #endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) #if defined(__SSE4_1__) @@ -1480,246 +518,6 @@ struct SquaredEuclideanDistanceMatrix { float *out); }; -/*! Squared Euclidean Distance Matrix (INT8, M=2, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=2, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=4, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=4, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=4, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=8, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=8, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=8, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=8, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=16, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=16, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=16, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=16, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=16, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT8, M=32, N=32) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Euclidean Distance Matrix (INT8, M=1, N=1) */ template <> @@ -1732,246 +530,6 @@ struct EuclideanDistanceMatrix { float *out); }; -/*! Euclidean Distance Matrix (INT8, M=2, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=2, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=4, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=4, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=4, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=8, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=8, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=8, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=8, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=16, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=16, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=16, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=16, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=16, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT8, M=32, N=32) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Squared Euclidean Distance Matrix (INT4, M=1, N=1) */ template <> @@ -1984,246 +542,6 @@ struct SquaredEuclideanDistanceMatrix { float *out); }; -/*! Squared Euclidean Distance Matrix (INT4, M=2, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=2, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=4, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=4, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=4, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=8, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=8, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=8, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=8, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=16, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=16, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=16, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=16, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=16, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=1) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=2) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=4) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=8) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=16) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Squared Euclidean Distance Matrix (INT4, M=32, N=32) - */ -template <> -struct SquaredEuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Euclidean Distance Matrix (INT4, M=1, N=1) */ template <> @@ -2235,246 +553,6 @@ struct EuclideanDistanceMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -/*! Euclidean Distance Matrix (INT4, M=2, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=2, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=4, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=4, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=4, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=8, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=8, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=8, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=8, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=16, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=16, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=16, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=16, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=16, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=1) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=2) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=4) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=8) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=16) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Euclidean Distance Matrix (INT4, M=32, N=32) - */ -template <> -struct EuclideanDistanceMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; #endif // __SSE4_1__ /*! Squared Euclidean Distance Sparse Matrix diff --git a/src/ailego/math/euclidean_distance_matrix_fp16.cc b/src/ailego/math/euclidean_distance_matrix_fp16.cc deleted file mode 100644 index ca24561..0000000 --- a/src/ailego/math/euclidean_distance_matrix_fp16.cc +++ /dev/null @@ -1,615 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "distance_matrix_accum_fp16.i" -#include "euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE -#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX -#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512 -#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON -#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL -#define ACCUM_FP16_STEP_NEON SSD_FP16_NEON - -//! Calculate sum of squared difference (SSE) -#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ - { \ - __m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \ - xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \ - } - -//! Calculate sum of squared difference (AVX) -#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ - { \ - __m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \ - ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \ - } - -//! Calculate sum of squared difference (AVX512) -#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - { \ - __m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \ - zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \ - } - -//! Calculate sum of squared difference (GENERAL) -#define SSD_FP16_GENERAL(m, q, sum) \ - { \ - float x = m - q; \ - sum += (x * x); \ - } - -//! Calculate sum of squared difference (NEON) -#define SSD_FP16_NEON(v_m, v_q, v_sum) \ - { \ - float16x8_t v_d = vsubq_f16(v_m, v_q); \ - v_sum = vfmaq_f16(v_sum, v_d, v_d); \ - } - -//! Calculate sum of squared difference (NEON) -#define SSD_FP32_NEON(v_m, v_q, v_sum) \ - { \ - float32x4_t v_d = vsubq_f32(v_m, v_q); \ - v_sum = vfmaq_f32(v_sum, v_d, v_d); \ - } - -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) - -#if defined(__AVX512FP16__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, - const Float16 *rhs, - size_t size) { - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 6) << 6); - - __m512h zmm_sum_0 = _mm512_setzero_ph(); - __m512h zmm_sum_1 = _mm512_setzero_ph(); - - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m512h zmm_d_0 = - _mm512_sub_ph(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0)); - __m512h zmm_d_1 = - _mm512_sub_ph(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32)); - zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0); - zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1); - } - - if (last >= last_aligned + 32) { - __m512h zmm_d = _mm512_sub_ph(_mm512_load_ph(lhs), _mm512_load_ph(rhs)); - zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0); - lhs += 32; - rhs += 32; - } - } else { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m512h zmm_d_0 = - _mm512_sub_ph(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0)); - __m512h zmm_d_1 = - _mm512_sub_ph(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32)); - zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0); - zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1); - } - - if (last >= last_aligned + 32) { - __m512h zmm_d = _mm512_sub_ph(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs)); - zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0); - lhs += 32; - rhs += 32; - } - } - - zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1); - if (lhs != last) { - __mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1); - __m512i zmm_undefined = _mm512_undefined_epi32(); - __m512h zmm_undefined_ph = _mm512_undefined_ph(); - __m512h zmm_d = _mm512_mask_sub_ph( - zmm_undefined_ph, mask, - _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)), - _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs))); - zmm_sum_0 = _mm512_mask3_fmadd_ph(zmm_d, zmm_d, zmm_sum_0, mask); - } - - return HorizontalAdd_FP16_V512(zmm_sum_0); -} -#endif - - -//! Compute the distance between matrix and query (FP16, M=1, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, ) -#else -#if defined(__AVX512FP16__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { - *out = SquaredEuclideanDistanceAVX512FP16(m, q, dim); - return; - } -#endif -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) - return; - } -#endif - ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, ) -#endif //__ARM_NEON -} - -//! Compute the distance between matrix and query (FP16, M=1, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, std::sqrt) -#else -#if defined(__AVX512FP16__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { - *out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim)); - return; - } -#endif -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt) - return; - } -#endif - ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, std::sqrt) -#endif //__ARM_NEON -} - -#if !defined(__ARM_NEON) -//! Compute the distance between matrix and query (FP16, M=2, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_2X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=2, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_2X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_4X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_4X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_4X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_8X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_8X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_8X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { - ACCUM_FP16_8X8_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X1_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - - ACCUM_FP16_16X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X2_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - - ACCUM_FP16_16X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X4_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X8_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X8_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=16) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X16_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X16_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X1_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X2_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X4_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X8_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X8_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=16) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X16_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X16_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=32) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X32_AVX512(m, q, dim, out, ) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X32_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=2, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_2X1_AVX(m, q, dim, out, _mm_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=2, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_2X2_AVX(m, q, dim, out, _mm_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X1_AVX(m, q, dim, out, _mm_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X2_AVX(m, q, dim, out, _mm_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X4_AVX(m, q, dim, out, _mm_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps) -} - -//! Compute the distance between matrix and query (FP16, M=32, N=32) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps) - return; - } -#endif // __AVX512F__ - ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps) -} -#endif // !__ARM_NEON -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp16_avx.cc b/src/ailego/math/euclidean_distance_matrix_fp16_avx.cc new file mode 100644 index 0000000..0adf738 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp16_avx.cc @@ -0,0 +1,38 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) + +void SquaredEuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out) { + ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, ) +} + +//! EuclideanDistance +void EuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out) { + ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, std::sqrt) +} + +#endif // __AVX__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp16_avx512.cc b/src/ailego/math/euclidean_distance_matrix_fp16_avx512.cc new file mode 100644 index 0000000..244f5db --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp16_avx512.cc @@ -0,0 +1,96 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512FP16__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, const Float16 *rhs, + size_t size) { + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 6) << 6); + + __m512h zmm_sum_0 = _mm512_setzero_ph(); + __m512h zmm_sum_1 = _mm512_setzero_ph(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m512h zmm_d_0 = + _mm512_sub_ph(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0)); + __m512h zmm_d_1 = + _mm512_sub_ph(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32)); + zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 32) { + __m512h zmm_d = _mm512_sub_ph(_mm512_load_ph(lhs), _mm512_load_ph(rhs)); + zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0); + lhs += 32; + rhs += 32; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m512h zmm_d_0 = + _mm512_sub_ph(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0)); + __m512h zmm_d_1 = + _mm512_sub_ph(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32)); + zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 32) { + __m512h zmm_d = _mm512_sub_ph(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs)); + zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0); + lhs += 32; + rhs += 32; + } + } + + zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1); + __m512i zmm_undefined = _mm512_undefined_epi32(); + __m512h zmm_undefined_ph = _mm512_undefined_ph(); + __m512h zmm_d = _mm512_mask_sub_ph( + zmm_undefined_ph, mask, + _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)), + _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs))); + zmm_sum_0 = _mm512_mask3_fmadd_ph(zmm_d, zmm_d, zmm_sum_0, mask); + } + + return HorizontalAdd_FP16_V512(zmm_sum_0); +} +#endif + +#if defined(__AVX512F__) +void SquaredEuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out) { + ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, ) +} + +//! EuclideanDistance +void EuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out) { + ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, std::sqrt) +} + +#endif +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp16_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp16_dispatch.cc new file mode 100644 index 0000000..1d08b8b --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp16_dispatch.cc @@ -0,0 +1,87 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +void SquaredEuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out); +void EuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out); +#endif + +#if defined(__AVX512FP16__) +float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, const Float16 *rhs, + size_t size); +#endif + +#if defined(__AVX512F__) +void SquaredEuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out); + +void EuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out); +#endif + +#if defined(__AVX__) +void SquaredEuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out); +void EuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out); +#endif + +#if (defined(__F16C__) && defined(__AVX__)) || \ + (defined(__ARM_NEON) && defined(__aarch64__)) +//! Compute the distance between matrix and query (FP16, M=1, N=1) +void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, + float *out) { +#if defined(__ARM_NEON) + SquaredEuclideanDistanceNEON(m, q, dim, out); +#else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = SquaredEuclideanDistanceAVX512FP16(m, q, dim); + return; + } +#endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + SquaredEuclideanDistanceAVX512(m, q, dim, out); + // ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) + return; + } +#endif + SquaredEuclideanDistanceAVX(m, q, dim, out); + // ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, ) +#endif //__ARM_NEON +} + +//! Compute the distance between matrix and query (FP16, M=1, N=1) +void EuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { + SquaredEuclideanDistanceMatrix::Compute(m, q, dim, out); + *out = std::sqrt(*out); +} + +#endif + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp16_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp16_neon.cc new file mode 100644 index 0000000..4527056 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp16_neon.cc @@ -0,0 +1,35 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +void SquaredEuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out) { + ACCUM_FP16_1X1_NEON(lhs, rhs, size, out, 0ull, ) +} + +void EuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out) { + ACCUM_FP16_1X1_NEON(lhs, rhs, size, out, 0ull, std::sqrt) +} +#endif + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp16_sse.cc b/src/ailego/math/euclidean_distance_matrix_fp16_sse.cc new file mode 100644 index 0000000..6291346 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp16_sse.cc @@ -0,0 +1,54 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "distance_matrix_accum_fp16.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE +#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL + +//! Calculate sum of squared difference (SSE) +#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ + { \ + __m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \ + xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \ + } + +//! Calculate sum of squared difference (GENERAL) +#define SSD_FP16_GENERAL(m, q, sum) \ + { \ + float x = m - q; \ + sum += (x * x); \ + } + +//! Calculate sum of squared difference (NEON) +#define SSD_FP16_NEON(v_m, v_q, v_sum) \ + { \ + float16x8_t v_d = vsubq_f16(v_m, v_q); \ + v_sum = vfmaq_f16(v_sum, v_d, v_d); \ + } + +//! Calculate sum of squared difference (NEON) +#define SSD_FP32_NEON(v_m, v_q, v_sum) \ + { \ + float32x4_t v_d = vsubq_f32(v_m, v_q); \ + v_sum = vfmaq_f32(v_sum, v_d, v_d); \ + } + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32.cc b/src/ailego/math/euclidean_distance_matrix_fp32.cc deleted file mode 100644 index 7a02473..0000000 --- a/src/ailego/math/euclidean_distance_matrix_fp32.cc +++ /dev/null @@ -1,930 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "distance_matrix_accum_fp32.i" -#include "euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE -#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX -#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512 -#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON - -//! Calculate sum of squared difference (GENERAL) -#define SSD_FP32_GENERAL(m, q, sum) \ - { \ - float x = m - q; \ - sum += (x * x); \ - } - -//! Calculate sum of squared difference (SSE) -#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ - { \ - __m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \ - xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \ - } - -//! Calculate sum of squared difference (AVX) -#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ - { \ - __m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \ - ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \ - } - -//! Calculate sum of squared difference (AVX512) -#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - { \ - __m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \ - zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \ - } - -//! Calculate sum of squared difference (NEON) -#define SSD_FP32_NEON(v_m, v_q, v_sum) \ - { \ - float32x4_t v_d = vsubq_f32(v_m, v_q); \ - v_sum = vfmaq_f32(v_sum, v_d, v_d); \ - } - -#if defined(__ARM_NEON) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceNEON(const float *lhs, - const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - float32x4_t v_sum_0 = vdupq_n_f32(0); - float32x4_t v_sum_1 = vdupq_n_f32(0); - - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - float32x4_t v_d_0 = vsubq_f32(vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); - float32x4_t v_d_1 = vsubq_f32(vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); - v_sum_0 = vfmaq_f32(v_sum_0, v_d_0, v_d_0); - v_sum_1 = vfmaq_f32(v_sum_1, v_d_1, v_d_1); - } - if (last >= last_aligned + 4) { - float32x4_t v_d = vsubq_f32(vld1q_f32(lhs), vld1q_f32(rhs)); - v_sum_0 = vfmaq_f32(v_sum_0, v_d, v_d); - lhs += 4; - rhs += 4; - } - - float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); - switch (last - lhs) { - case 3: - SSD_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __ARM_NEON - -#if defined(__SSE__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceSSE(const float *lhs, - const float *rhs, size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - __m128 xmm_sum_0 = _mm_setzero_ps(); - __m128 xmm_sum_1 = _mm_setzero_ps(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_d_0 = _mm_sub_ps(_mm_load_ps(lhs + 0), _mm_load_ps(rhs + 0)); - __m128 xmm_d_1 = _mm_sub_ps(_mm_load_ps(lhs + 4), _mm_load_ps(rhs + 4)); - xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); - xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); - } - - if (last >= last_aligned + 4) { - __m128 xmm_d = _mm_sub_ps(_mm_load_ps(lhs), _mm_load_ps(rhs)); - xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); - lhs += 4; - rhs += 4; - } - } else { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_d_0 = _mm_sub_ps(_mm_loadu_ps(lhs + 0), _mm_loadu_ps(rhs + 0)); - __m128 xmm_d_1 = _mm_sub_ps(_mm_loadu_ps(lhs + 4), _mm_loadu_ps(rhs + 4)); - xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); - xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); - } - - if (last >= last_aligned + 4) { - __m128 xmm_d = _mm_sub_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs)); - xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); - lhs += 4; - rhs += 4; - } - } - float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); - - switch (last - lhs) { - case 3: - SSD_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __SSE__ - -#if defined(__AVX__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceAVX(const float *lhs, - const float *rhs, size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 4) << 4); - - __m256 ymm_sum_0 = _mm256_setzero_ps(); - __m256 ymm_sum_1 = _mm256_setzero_ps(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_d_0 = - _mm256_sub_ps(_mm256_load_ps(lhs + 0), _mm256_load_ps(rhs + 0)); - __m256 ymm_d_1 = - _mm256_sub_ps(_mm256_load_ps(lhs + 8), _mm256_load_ps(rhs + 8)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); - } - - if (last >= last_aligned + 8) { - __m256 ymm_d = _mm256_sub_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); - lhs += 8; - rhs += 8; - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_d_0 = - _mm256_sub_ps(_mm256_loadu_ps(lhs + 0), _mm256_loadu_ps(rhs + 0)); - __m256 ymm_d_1 = - _mm256_sub_ps(_mm256_loadu_ps(lhs + 8), _mm256_loadu_ps(rhs + 8)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); - } - - if (last >= last_aligned + 8) { - __m256 ymm_d = _mm256_sub_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); - lhs += 8; - rhs += 8; - } - } - float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); - - switch (last - lhs) { - case 7: - SSD_FP32_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - SSD_FP32_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - SSD_FP32_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - SSD_FP32_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - SSD_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX__ - -#if defined(__AVX512F__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceAVX512(const float *lhs, - const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 5) << 5); - - __m512 zmm_sum_0 = _mm512_setzero_ps(); - __m512 zmm_sum_1 = _mm512_setzero_ps(); - - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512 zmm_d_0 = - _mm512_sub_ps(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0)); - __m512 zmm_d_1 = - _mm512_sub_ps(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16)); - zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); - zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m512 zmm_d = _mm512_sub_ps(_mm512_load_ps(lhs), _mm512_load_ps(rhs)); - zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512 zmm_d_0 = - _mm512_sub_ps(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0)); - __m512 zmm_d_1 = - _mm512_sub_ps(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16)); - zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); - zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m512 zmm_d = _mm512_sub_ps(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs)); - zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); - lhs += 16; - rhs += 16; - } - } - - zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); - if (lhs != last) { - __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); - __m512 zmm_undefined = _mm512_undefined_ps(); - __m512 zmm_d = _mm512_mask_sub_ps( - zmm_undefined, mask, _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), - _mm512_mask_loadu_ps(zmm_undefined, mask, rhs)); - zmm_sum_0 = _mm512_mask3_fmadd_ps(zmm_d, zmm_d, zmm_sum_0, mask); - } - return HorizontalAdd_FP32_V512(zmm_sum_0); -} -#endif - -#if defined(__SSE__) || defined(__ARM_NEON) -//! Compute the distance between matrix and query (FP32, M=1, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - *out = SquaredEuclideanDistanceNEON(m, q, dim); -#else -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - if (dim > 15) { - *out = SquaredEuclideanDistanceAVX512(m, q, dim); - return; - } - } -#endif // __AVX512F__ -#if defined(__AVX__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { - if (dim > 7) { - *out = SquaredEuclideanDistanceAVX(m, q, dim); - return; - } - } -#endif // __AVX__ - *out = SquaredEuclideanDistanceSSE(m, q, dim); -#endif // __ARM_NEON -} - -//! Compute the distance between matrix and query (FP32, M=2, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_2X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_2X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=2, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_2X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_2X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X4_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X4_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X4_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X4_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X8_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X8_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=16, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X1_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X1_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X1_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X2_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X2_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X2_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X4_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X4_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X4_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X8_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X8_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X8_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=16) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X16_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X16_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X16_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X16_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X1_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X1_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X1_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X2_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X2_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X2_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X4_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X4_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X4_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X8_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X8_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X8_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=16) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X16_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X16_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X16_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X16_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=32) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X32_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X32_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X32_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X32_SSE(m, q, dim, out, ) -#endif -} -#endif // __SSE__ || __ARM_NEON - -#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) -//! Compute the distance between matrix and query (FP32, M=1, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - *out = std::sqrt(SquaredEuclideanDistanceNEON(m, q, dim)); -#else -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - if (dim > 15) { - *out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim)); - return; - } - } -#endif // __AVX512F__ - -#if defined(__AVX__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { - if (dim > 7) { - *out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim)); - return; - } - } -#endif // __AVX__ - *out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim)); -#endif // __ARM_NEON -} - -//! Compute the distance between matrix and query (FP32, M=2, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X1_NEON(m, q, dim, out, vsqrt_f32) -#elif defined(__AVX__) - ACCUM_FP32_2X1_AVX(m, q, dim, out, _mm_sqrt_ps) -#else - ACCUM_FP32_2X1_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=2, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X2_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_2X2_AVX(m, q, dim, out, _mm_sqrt_ps) -#else - ACCUM_FP32_2X2_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X1_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X1_AVX(m, q, dim, out, _mm_sqrt_ps) -#else - ACCUM_FP32_4X1_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X2_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X2_AVX(m, q, dim, out, _mm_sqrt_ps) -#else - ACCUM_FP32_4X2_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X4_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X4_AVX(m, q, dim, out, _mm_sqrt_ps) -#else - ACCUM_FP32_4X4_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X1_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_8X1_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X2_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_8X2_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X4_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_8X4_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X8_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_8X8_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=16, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X1_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_16X1_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X2_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_16X2_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X4_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_16X4_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X8_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_16X8_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X16_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_16X16_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X1_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X1_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X2_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X2_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X4_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X4_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X8_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X8_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X16_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X16_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=32) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X32_NEON(m, q, dim, out, vsqrtq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#elif defined(__AVX__) - ACCUM_FP32_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps) -#else - ACCUM_FP32_32X32_SSE(m, q, dim, out, _mm_sqrt_ps) -#endif -} -#endif // __SSE__ || __ARM_NEON && __aarch64__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_avx.cc b/src/ailego/math/euclidean_distance_matrix_fp32_avx.cc new file mode 100644 index 0000000..3fdcad5 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp32_avx.cc @@ -0,0 +1,94 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) +float SquaredEuclideanDistanceAVX(const float *lhs, const float *rhs, + size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_d_0 = + _mm256_sub_ps(_mm256_load_ps(lhs + 0), _mm256_load_ps(rhs + 0)); + __m256 ymm_d_1 = + _mm256_sub_ps(_mm256_load_ps(lhs + 8), _mm256_load_ps(rhs + 8)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + __m256 ymm_d = _mm256_sub_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_d_0 = + _mm256_sub_ps(_mm256_loadu_ps(lhs + 0), _mm256_loadu_ps(rhs + 0)); + __m256 ymm_d_1 = + _mm256_sub_ps(_mm256_loadu_ps(lhs + 8), _mm256_loadu_ps(rhs + 8)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + __m256 ymm_d = _mm256_sub_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0); + lhs += 8; + rhs += 8; + } + } + float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + + switch (last - lhs) { + case 7: + SSD_FP32_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + SSD_FP32_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + SSD_FP32_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + SSD_FP32_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + SSD_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_FP32_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __AVX__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_avx512.cc b/src/ailego/math/euclidean_distance_matrix_fp32_avx512.cc new file mode 100644 index 0000000..f9a8250 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp32_avx512.cc @@ -0,0 +1,81 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512F__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceAVX512(const float *lhs, const float *rhs, + size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_d_0 = + _mm512_sub_ps(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0)); + __m512 zmm_d_1 = + _mm512_sub_ps(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m512 zmm_d = _mm512_sub_ps(_mm512_load_ps(lhs), _mm512_load_ps(rhs)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_d_0 = + _mm512_sub_ps(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0)); + __m512 zmm_d_1 = + _mm512_sub_ps(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0); + zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m512 zmm_d = _mm512_sub_ps(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs)); + zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0); + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + __m512 zmm_d = _mm512_mask_sub_ps( + zmm_undefined, mask, _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), + _mm512_mask_loadu_ps(zmm_undefined, mask, rhs)); + zmm_sum_0 = _mm512_mask3_fmadd_ps(zmm_d, zmm_d, zmm_sum_0, mask); + } + return HorizontalAdd_FP32_V512(zmm_sum_0); +} + +#endif + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc new file mode 100644 index 0000000..08d31c6 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp32_dispatch.cc @@ -0,0 +1,92 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +void SquaredEuclideanDistanceNEON(const float *lhs, const float *rhs, + size_t size, float *out); +#endif + +#if defined(__AVX512F__) +float SquaredEuclideanDistanceAVX512(const float *lhs, const float *rhs, + size_t size); +float EuclideanDistanceAVX512(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__AVX__) +float SquaredEuclideanDistanceAVX(const float *lhs, const float *rhs, + size_t size); +float EuclideanDistanceAVX(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__SSE__) +float SquaredEuclideanDistanceSSE(const float *lhs, const float *rhs, + size_t size); +float EuclideanDistanceSSE(const float *lhs, const float *rhs, size_t size); +#endif + +//----------------------------------------------------------- +// SquaredEuclideanDistance +//----------------------------------------------------------- +#if defined(__SSE__) || defined(__ARM_NEON) +//! Compute the distance between matrix and query (FP32, M=1, N=1) +void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, + float *out) { +#if defined(__ARM_NEON) + SquaredEuclideanDistanceNEON(m, q, dim, out); +#else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = SquaredEuclideanDistanceAVX512(m, q, dim); + return; + } + } +#endif // __AVX512F__ +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = SquaredEuclideanDistanceAVX(m, q, dim); + return; + } + } +#endif // __AVX__ + *out = SquaredEuclideanDistanceSSE(m, q, dim); +#endif // __ARM_NEON +} +#endif // __SSE__ || __ARM_NEON + + +//----------------------------------------------------------- +// EuclideanDistance +//----------------------------------------------------------- +#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__)) +//! Compute the distance between matrix and query (FP32, M=1, N=1) +void EuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { + SquaredEuclideanDistanceMatrix::Compute(m, q, dim, out); + *out = std::sqrt(*out); +} +#endif // __SSE__ || __ARM_NEON && __aarch64__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc new file mode 100644 index 0000000..3827faf --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp32_neon.cc @@ -0,0 +1,62 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +//! Squared Euclidean Distance +void SquaredEuclideanDistanceNEON(const float *lhs, const float *rhs, + size_t size, float *out) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float32x4_t v_d_0 = vsubq_f32(vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); + float32x4_t v_d_1 = vsubq_f32(vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); + v_sum_0 = vfmaq_f32(v_sum_0, v_d_0, v_d_0); + v_sum_1 = vfmaq_f32(v_sum_1, v_d_1, v_d_1); + } + if (last >= last_aligned + 4) { + float32x4_t v_d = vsubq_f32(vld1q_f32(lhs), vld1q_f32(rhs)); + v_sum_0 = vfmaq_f32(v_sum_0, v_d, v_d); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + switch (last - lhs) { + case 3: + SSD_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_FP32_GENERAL(lhs[0], rhs[0], result) + } + *out = result; +} + +#endif // __ARM_NEON + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_fp32_sse.cc b/src/ailego/math/euclidean_distance_matrix_fp32_sse.cc new file mode 100644 index 0000000..a4cf588 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_fp32_sse.cc @@ -0,0 +1,78 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE__) +float SquaredEuclideanDistanceSSE(const float *lhs, const float *rhs, + size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum_0 = _mm_setzero_ps(); + __m128 xmm_sum_1 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_d_0 = _mm_sub_ps(_mm_load_ps(lhs + 0), _mm_load_ps(rhs + 0)); + __m128 xmm_d_1 = _mm_sub_ps(_mm_load_ps(lhs + 4), _mm_load_ps(rhs + 4)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + __m128 xmm_d = _mm_sub_ps(_mm_load_ps(lhs), _mm_load_ps(rhs)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_d_0 = _mm_sub_ps(_mm_loadu_ps(lhs + 0), _mm_loadu_ps(rhs + 0)); + __m128 xmm_d_1 = _mm_sub_ps(_mm_loadu_ps(lhs + 4), _mm_loadu_ps(rhs + 4)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + __m128 xmm_d = _mm_sub_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs)); + xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0); + lhs += 4; + rhs += 4; + } + } + float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); + + switch (last - lhs) { + case 3: + SSD_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_FP32_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __SSE__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int4.cc b/src/ailego/math/euclidean_distance_matrix_int4.cc deleted file mode 100644 index bef4321..0000000 --- a/src/ailego/math/euclidean_distance_matrix_int4.cc +++ /dev/null @@ -1,801 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_int4.i" -#include "euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_INT4_STEP_SSE SSD_INT4_SSE -#define ACCUM_INT4_STEP_AVX SSD_INT4_AVX - -#if defined(__SSE4_1__) -static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0xf0f0f0f0); -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0xf0f0f0f0); -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -#endif // __AVX2__ - -//! Calculate sum of squared difference (GENERAL) -#define SSD_INT4_GENERAL(m, q, sum) \ - sum += Int4SquaredDiffTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \ - Int4SquaredDiffTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)]; - -//! Calculate sum of squared difference (SSE) -#define SSD_INT4_SSE(xmm_m, xmm_q, xmm_sum) \ - { \ - __m128i xmm_lhs = \ - _mm_and_si128(_mm_slli_epi32((xmm_m), 4), MASK_INT4_SSE); \ - __m128i xmm_rhs = \ - _mm_and_si128(_mm_slli_epi32((xmm_q), 4), MASK_INT4_SSE); \ - xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \ - _mm_min_epi8(xmm_lhs, xmm_rhs)), \ - 4); \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \ - xmm_sum); \ - xmm_lhs = _mm_and_si128((xmm_m), MASK_INT4_SSE); \ - xmm_rhs = _mm_and_si128((xmm_q), MASK_INT4_SSE); \ - xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \ - _mm_min_epi8(xmm_lhs, xmm_rhs)), \ - 4); \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \ - xmm_sum); \ - } - -//! Calculate sum of squared difference (AVX) -#define SSD_INT4_AVX(ymm_m, ymm_q, ymm_sum) \ - { \ - __m256i ymm_lhs = \ - _mm256_and_si256(_mm256_slli_epi32((ymm_m), 4), MASK_INT4_AVX); \ - __m256i ymm_rhs = \ - _mm256_and_si256(_mm256_slli_epi32((ymm_q), 4), MASK_INT4_AVX); \ - ymm_lhs = \ - _mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \ - _mm256_min_epi8(ymm_lhs, ymm_rhs)), \ - 4); \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \ - ONES_INT16_AVX), \ - ymm_sum); \ - ymm_lhs = _mm256_and_si256((ymm_m), MASK_INT4_AVX); \ - ymm_rhs = _mm256_and_si256((ymm_q), MASK_INT4_AVX); \ - ymm_lhs = \ - _mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \ - _mm256_min_epi8(ymm_lhs, ymm_rhs)), \ - 4); \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \ - ONES_INT16_AVX), \ - ymm_sum); \ - } - -//! Compute the distance between matrix and query -#define SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ - { \ - __m128i xmm_lhs_0 = \ - _mm_and_si128(_mm_slli_epi32((xmm_lhs), 4), MASK_INT4_SSE); \ - __m128i xmm_rhs_0 = \ - _mm_and_si128(_mm_slli_epi32((xmm_rhs), 4), MASK_INT4_SSE); \ - __m128i xmm_lhs_1 = _mm_and_si128((xmm_lhs), MASK_INT4_SSE); \ - __m128i xmm_rhs_1 = _mm_and_si128((xmm_rhs), MASK_INT4_SSE); \ - xmm_lhs_0 = \ - _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), \ - _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)), \ - 4); \ - xmm_rhs_0 = \ - _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), \ - _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)), \ - 4); \ - xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs_0, xmm_lhs_0), \ - ONES_INT16_SSE); \ - xmm_rhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_rhs_0), \ - ONES_INT16_SSE); \ - xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_rhs_0), xmm_sum); \ - } - -//! Compute the distance between matrix and query -#define SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ - { \ - __m256i ymm_lhs_0 = \ - _mm256_and_si256(_mm256_slli_epi32((ymm_lhs), 4), MASK_INT4_AVX); \ - __m256i ymm_rhs_0 = \ - _mm256_and_si256(_mm256_slli_epi32((ymm_rhs), 4), MASK_INT4_AVX); \ - __m256i ymm_lhs_1 = _mm256_and_si256((ymm_lhs), MASK_INT4_AVX); \ - __m256i ymm_rhs_1 = _mm256_and_si256((ymm_rhs), MASK_INT4_AVX); \ - ymm_lhs_0 = _mm256_srli_epi32( \ - _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), \ - _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)), \ - 4); \ - ymm_rhs_0 = _mm256_srli_epi32( \ - _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), \ - _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)), \ - 4); \ - ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs_0, ymm_lhs_0), \ - ONES_INT16_AVX); \ - ymm_rhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_rhs_0), \ - ONES_INT16_AVX); \ - ymm_sum = \ - _mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_rhs_0), ymm_sum); \ - } - -//! Compute the square root of value (SSE) -#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v)) - -//! Compute the square root of value (AVX) -#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v)) - -//! Compute the square root of value (AVX512) -#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v)) - -#if defined(__SSE4_1__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceSSE(const uint8_t *lhs, - const uint8_t *rhs, - size_t size) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 4) << 4); - - __m128i xmm_sum = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); - SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); - SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - } - } - float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); - - switch (last - lhs) { - case 15: - SSD_INT4_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - SSD_INT4_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - SSD_INT4_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - SSD_INT4_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - SSD_INT4_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - SSD_INT4_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - SSD_INT4_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - SSD_INT4_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - SSD_INT4_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - SSD_INT4_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - SSD_INT4_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - SSD_INT4_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - SSD_INT4_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_INT4_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_INT4_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __SSE4_1__ - -#if defined(__AVX2__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceAVX(const uint8_t *lhs, - const uint8_t *rhs, - size_t size) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 5) << 5); - - __m256i ymm_sum = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); - SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) - } - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), - ymm_sum); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); - SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) - } - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), - ymm_sum); - lhs += 16; - rhs += 16; - } - } - float result = static_cast(HorizontalAdd_INT32_V256(ymm_sum)); - - switch (last - lhs) { - case 15: - SSD_INT4_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - SSD_INT4_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - SSD_INT4_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - SSD_INT4_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - SSD_INT4_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - SSD_INT4_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - SSD_INT4_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - SSD_INT4_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - SSD_INT4_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - SSD_INT4_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - SSD_INT4_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - SSD_INT4_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - SSD_INT4_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_INT4_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_INT4_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX2__ - -#if defined(__SSE4_1__) -//! Compute the distance between matrix and query (INT4, M=1, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - if (dim > 63) { - *out = SquaredEuclideanDistanceAVX(m, q, dim >> 1); - return; - } -#endif // __AVX2__ - *out = SquaredEuclideanDistanceSSE(m, q, dim >> 1); -} - -//! Compute the distance between matrix and query (INT4, M=2, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=2, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=16) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=16) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=32) -void SquaredEuclideanDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=1, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - if (dim > 63) { - *out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim >> 1)); - return; - } -#endif // __AVX2__ - *out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim >> 1)); -} - -//! Compute the distance between matrix and query (INT4, M=2, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X1_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT4_2X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=2, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X2_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT4_2X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X1_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT4_4X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X2_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT4_4X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X4_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT4_4X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_8X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_8X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_8X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_8X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_16X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_16X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_16X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_16X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X16_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_16X16_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X16_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X16_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=32) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X32_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT4_32X32_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int4_avx2.cc b/src/ailego/math/euclidean_distance_matrix_int4_avx2.cc new file mode 100644 index 0000000..0923249 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int4_avx2.cc @@ -0,0 +1,118 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int4.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs, + size_t size) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 5) << 5); + + __m256i ymm_sum = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); + SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) + } + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), + ymm_sum); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); + SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) + } + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), + ymm_sum); + lhs += 16; + rhs += 16; + } + } + float result = static_cast(HorizontalAdd_INT32_V256(ymm_sum)); + + switch (last - lhs) { + case 15: + SSD_INT4_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + SSD_INT4_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + SSD_INT4_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + SSD_INT4_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + SSD_INT4_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + SSD_INT4_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + SSD_INT4_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + SSD_INT4_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + SSD_INT4_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + SSD_INT4_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + SSD_INT4_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + SSD_INT4_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + SSD_INT4_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_INT4_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_INT4_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __AVX2__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int4_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_int4_dispatch.cc new file mode 100644 index 0000000..beeb7a2 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int4_dispatch.cc @@ -0,0 +1,60 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +float SquaredEuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs, + size_t size); +float EuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs, + size_t size); +#endif + +#if defined(__SSE4_1__) +float SquaredEuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs, + size_t size); +float EuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size); +#endif + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query (INT4, M=1, N=1) +void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, + float *out) { +#if defined(__AVX2__) + if (dim > 63) { + *out = SquaredEuclideanDistanceAVX2(m, q, dim >> 1); + return; + } +#endif // __AVX2__ + *out = SquaredEuclideanDistanceSSE(m, q, dim >> 1); +} + +//! Compute the distance between matrix and query (INT4, M=1, N=1) +void EuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { + SquaredEuclideanDistanceMatrix::Compute(m, q, dim, out); + *out = std::sqrt(*out); +} + +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int4_sse.cc b/src/ailego/math/euclidean_distance_matrix_int4_sse.cc new file mode 100644 index 0000000..63e10da --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int4_sse.cc @@ -0,0 +1,98 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int4.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs, + size_t size) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 4) << 4); + + __m128i xmm_sum = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); + SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); + SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + } + } + float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); + + switch (last - lhs) { + case 15: + SSD_INT4_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + SSD_INT4_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + SSD_INT4_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + SSD_INT4_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + SSD_INT4_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + SSD_INT4_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + SSD_INT4_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + SSD_INT4_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + SSD_INT4_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + SSD_INT4_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + SSD_INT4_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + SSD_INT4_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + SSD_INT4_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_INT4_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_INT4_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int8.cc b/src/ailego/math/euclidean_distance_matrix_int8.cc deleted file mode 100644 index cd14d2d..0000000 --- a/src/ailego/math/euclidean_distance_matrix_int8.cc +++ /dev/null @@ -1,884 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_int8.i" -#include "euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_INT8_STEP_SSE SSD_INT8_SSE -#define ACCUM_INT8_STEP_AVX SSD_INT8_AVX - -#if defined(__SSE4_1__) -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -#endif // __AVX2__ - -//! Calculate sum of squared difference (GENERAL) -#define SSD_INT8_GENERAL(m, q, sum) \ - { \ - int32_t x = m - q; \ - sum += static_cast(x * x); \ - } - -//! Calculate sum of squared difference (SSE) -#define SSD_INT8_SSE(xmm_m, xmm_q, xmm_sum) \ - { \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_m), \ - _mm_sign_epi8(xmm_m, xmm_m)), \ - ONES_INT16_SSE), \ - xmm_sum); \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \ - _mm_sign_epi8(xmm_q, xmm_q)), \ - ONES_INT16_SSE), \ - xmm_sum); \ - xmm_sum = _mm_sub_epi32( \ - xmm_sum, \ - _mm_slli_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \ - _mm_sign_epi8(xmm_m, xmm_q)), \ - ONES_INT16_SSE), \ - 1)); \ - } - -//! Calculate sum of squared difference (AVX) -#define SSD_INT8_AVX(ymm_m, ymm_q, ymm_sum) \ - { \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_m), \ - _mm256_sign_epi8(ymm_m, ymm_m)), \ - ONES_INT16_AVX), \ - ymm_sum); \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ - _mm256_sign_epi8(ymm_q, ymm_q)), \ - ONES_INT16_AVX), \ - ymm_sum); \ - ymm_sum = _mm256_sub_epi32( \ - ymm_sum, _mm256_slli_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ - _mm256_sign_epi8(ymm_m, ymm_q)), \ - ONES_INT16_AVX), \ - 1)); \ - } - -//! Compute the square root of value (SSE) -#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v)) - -//! Compute the square root of value (AVX) -#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v)) - -//! Compute the square root of value (AVX512) -#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v)) - -#if defined(__SSE4_1__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceSSE(const int8_t *lhs, - const int8_t *rhs, - size_t size) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 5) << 5); - - __m128i xmm_sum_0 = _mm_setzero_si128(); - __m128i xmm_sum_1 = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); - - __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), - _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)); - xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), - _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)); - xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1); - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), - _mm_min_epi8(xmm_lhs, xmm_rhs)); - xmm_lhs = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0); - xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); - - __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), - _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)); - xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), - _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)); - xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1); - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), - _mm_min_epi8(xmm_lhs, xmm_rhs)); - xmm_lhs = _mm_cvtepu8_epi16(xmm_d); - xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); - xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0); - xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1); - lhs += 16; - rhs += 16; - } - } - float result = static_cast( - HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1))); - - switch (last - lhs) { - case 15: - SSD_INT8_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - SSD_INT8_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - SSD_INT8_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - SSD_INT8_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - SSD_INT8_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - SSD_INT8_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - SSD_INT8_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - SSD_INT8_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - SSD_INT8_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - SSD_INT8_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - SSD_INT8_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - SSD_INT8_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - SSD_INT8_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_INT8_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_INT8_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __SSE4_1__ - -#if defined(__AVX2__) -//! Squared Euclidean Distance -static inline float SquaredEuclideanDistanceAVX(const int8_t *lhs, - const int8_t *rhs, - size_t size) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 6) << 6); - float result = 0.0; - - __m256i ymm_sum_0 = _mm256_setzero_si256(); - __m256i ymm_sum_1 = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); - - __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), - _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)); - ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1); - - ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), - _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)); - ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); - __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), - _mm256_min_epi8(ymm_lhs, ymm_rhs)); - ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1); - lhs += 32; - rhs += 32; - } - } else { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); - - __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), - _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)); - ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1); - - ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), - _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)); - ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); - __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), - _mm256_min_epi8(ymm_lhs, ymm_rhs)); - ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); - ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); - ymm_sum_0 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0); - ymm_sum_1 = - _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1); - lhs += 32; - rhs += 32; - } - } - result = static_cast( - HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), - _mm_min_epi8(xmm_lhs, xmm_rhs)); - xmm_lhs = _mm_cvtepu8_epi16(xmm_sum); - xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_sum, xmm_sum)); - xmm_sum = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), - _mm_madd_epi16(xmm_rhs, xmm_rhs)); - result += static_cast(HorizontalAdd_INT32_V128(xmm_sum)); - lhs += 16; - rhs += 16; - } - switch (last - lhs) { - case 15: - SSD_INT8_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - SSD_INT8_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - SSD_INT8_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - SSD_INT8_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - SSD_INT8_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - SSD_INT8_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - SSD_INT8_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - SSD_INT8_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - SSD_INT8_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - SSD_INT8_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - SSD_INT8_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - SSD_INT8_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - SSD_INT8_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - SSD_INT8_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - SSD_INT8_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX2__ - -#if defined(__SSE4_1__) -//! Compute the distance between matrix and query (INT8, M=1, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - if (dim > 31) { - *out = SquaredEuclideanDistanceAVX(m, q, dim); - return; - } -#endif // __AVX2__ - *out = SquaredEuclideanDistanceSSE(m, q, dim); -} - -//! Compute the distance between matrix and query (INT8, M=2, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=2, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=16) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=1) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=2) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=4) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=8) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=16) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=32) -void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=1, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - if (dim > 31) { - *out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim)); - return; - } -#endif // __AVX2__ - *out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim)); -} - -//! Compute the distance between matrix and query (INT8, M=2, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X1_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT8_2X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=2, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X2_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT8_2X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X1_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT8_4X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X2_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT8_4X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X4_AVX(m, q, dim, out, SQRT_FP32_SSE) -#else - ACCUM_INT8_4X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_8X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_8X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_8X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_8X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_16X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_16X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_16X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_16X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X16_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_16X16_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=1) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X1_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X1_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=2) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X2_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X2_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=4) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X4_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X4_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=8) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X8_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X8_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=16) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X16_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X16_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=32) -void EuclideanDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X32_AVX(m, q, dim, out, SQRT_FP32_AVX) -#else - ACCUM_INT8_32X32_SSE(m, q, dim, out, SQRT_FP32_SSE) -#endif // __AVX2__ -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int8_avx2.cc b/src/ailego/math/euclidean_distance_matrix_int8_avx2.cc new file mode 100644 index 0000000..014281c --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int8_avx2.cc @@ -0,0 +1,182 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs, + size_t size) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 6) << 6); + float result = 0.0; + + __m256i ymm_sum_0 = _mm256_setzero_si256(); + __m256i ymm_sum_1 = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); + + __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), + _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)); + ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1); + + ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), + _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)); + ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); + __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), + _mm256_min_epi8(ymm_lhs, ymm_rhs)); + ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1); + lhs += 32; + rhs += 32; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); + + __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), + _mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)); + ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1); + + ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), + _mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)); + ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); + __m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), + _mm256_min_epi8(ymm_lhs, ymm_rhs)); + ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d)); + ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1)); + ymm_sum_0 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0); + ymm_sum_1 = + _mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1); + lhs += 32; + rhs += 32; + } + } + result = static_cast( + HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), + _mm_min_epi8(xmm_lhs, xmm_rhs)); + xmm_lhs = _mm_cvtepu8_epi16(xmm_sum); + xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_sum, xmm_sum)); + xmm_sum = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), + _mm_madd_epi16(xmm_rhs, xmm_rhs)); + result += static_cast(HorizontalAdd_INT32_V128(xmm_sum)); + lhs += 16; + rhs += 16; + } + switch (last - lhs) { + case 15: + SSD_INT8_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + SSD_INT8_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + SSD_INT8_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + SSD_INT8_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + SSD_INT8_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + SSD_INT8_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + SSD_INT8_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + SSD_INT8_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + SSD_INT8_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + SSD_INT8_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + SSD_INT8_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + SSD_INT8_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + SSD_INT8_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_INT8_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_INT8_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __AVX2__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int8_dispatch.cc b/src/ailego/math/euclidean_distance_matrix_int8_dispatch.cc new file mode 100644 index 0000000..54e9a75 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int8_dispatch.cc @@ -0,0 +1,59 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +float SquaredEuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs, + size_t size); +float EuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs, size_t size); +#endif + +#if defined(__SSE4_1__) +float SquaredEuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs, + size_t size); +float EuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs, size_t size); +#endif + + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query (INT8, M=1, N=1) +void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, + float *out) { +#if defined(__AVX2__) + if (dim > 31) { + *out = SquaredEuclideanDistanceAVX2(m, q, dim); + return; + } +#endif // __AVX2__ + *out = SquaredEuclideanDistanceSSE(m, q, dim); +} + +//! Compute the distance between matrix and query (INT8, M=1, N=1) +void EuclideanDistanceMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { + SquaredEuclideanDistanceMatrix::Compute(m, q, dim, out); + *out = std::sqrt(*out); +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/euclidean_distance_matrix_int8_sse.cc b/src/ailego/math/euclidean_distance_matrix_int8_sse.cc new file mode 100644 index 0000000..ca18ae9 --- /dev/null +++ b/src/ailego/math/euclidean_distance_matrix_int8_sse.cc @@ -0,0 +1,164 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_euclidean_utility.i" +#include "euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Squared Euclidean Distance +float SquaredEuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs, + size_t size) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 5) << 5); + + __m128i xmm_sum_0 = _mm_setzero_si128(); + __m128i xmm_sum_1 = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); + + __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), + _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)); + xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), + _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)); + xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1); + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), + _mm_min_epi8(xmm_lhs, xmm_rhs)); + xmm_lhs = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0); + xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); + + __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), + _mm_min_epi8(xmm_lhs_0, xmm_rhs_0)); + xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), + _mm_min_epi8(xmm_lhs_1, xmm_rhs_1)); + xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1); + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + __m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), + _mm_min_epi8(xmm_lhs, xmm_rhs)); + xmm_lhs = _mm_cvtepu8_epi16(xmm_d); + xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d)); + xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0); + xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1); + lhs += 16; + rhs += 16; + } + } + float result = static_cast( + HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1))); + + switch (last - lhs) { + case 15: + SSD_INT8_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + SSD_INT8_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + SSD_INT8_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + SSD_INT8_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + SSD_INT8_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + SSD_INT8_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + SSD_INT8_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + SSD_INT8_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + SSD_INT8_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + SSD_INT8_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + SSD_INT8_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + SSD_INT8_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + SSD_INT8_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + SSD_INT8_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + SSD_INT8_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/hamming_distance_matrix.cc b/src/ailego/math/hamming_distance_matrix.cc index 0e990d4..5009a78 100644 --- a/src/ailego/math/hamming_distance_matrix.cc +++ b/src/ailego/math/hamming_distance_matrix.cc @@ -314,248 +314,6 @@ void HammingDistanceMatrix::Compute(const ValueType *m, *out = static_cast(HammingDistance(m, q, cnt)); } -#if defined(__SSSE3__) -//! Compute the distance between matrix and query (UINT32, M=2, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_2X1_AVX(m, q, cnt, out, _mm_cvtepi32_ps) -#else - POPCNT_UINT32_2X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=2, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_2X2_AVX(m, q, cnt, out, _mm_cvtepi32_ps) -#else - POPCNT_UINT32_2X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X1_AVX(m, q, cnt, out, _mm_cvtepi32_ps) -#else - POPCNT_UINT32_4X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X2_AVX(m, q, cnt, out, _mm_cvtepi32_ps) -#else - POPCNT_UINT32_4X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X4_AVX(m, q, cnt, out, _mm_cvtepi32_ps) -#else - POPCNT_UINT32_4X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_8X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_8X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_8X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_8X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_16X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_16X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_16X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_16X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=16) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X16_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_16X16_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=16) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X16_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X16_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=32) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X32_AVX(m, q, cnt, out, _mm256_cvtepi32_ps) -#else - POPCNT_UINT32_32X32_SSE(m, q, cnt, out, _mm_cvtepi32_ps) -#endif -} -#endif // __SSSE3__ - #if defined(AILEGO_M64) //! Compute the distance between matrix and query (UINT64, M=1, N=1) void HammingDistanceMatrix::Compute(const ValueType *m, @@ -571,167 +329,6 @@ void HammingDistanceMatrix::Compute(const ValueType *m, *out = static_cast(HammingDistance(m, q, cnt)); } -#if defined(__AVX2__) -//! Compute the distance between matrix and query (UINT64, M=2, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_2X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=2, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_2X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=16) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X16_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=1) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=2) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=4) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=8) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=16) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X16_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=32) -void HammingDistanceMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X32_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32) -} -#endif // __AVX2__ #endif // AILEGO_M64 //! Compute the distance between matrix and query (UINT32, M=1, N=1) @@ -747,227 +344,6 @@ void HammingSquareRootDistanceMatrix::Compute( *out = std::sqrt(static_cast(HammingDistance(m, q, cnt))); } -#if defined(__SSSE3__) -//! Compute the distance between matrix and query (UINT32, M=2, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_2X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#else - POPCNT_UINT32_2X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=2, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_2X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#else - POPCNT_UINT32_2X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#else - POPCNT_UINT32_4X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#else - POPCNT_UINT32_4X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=4, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_4X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#else - POPCNT_UINT32_4X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_8X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_8X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_8X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=8, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_8X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_8X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_16X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_16X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_16X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_16X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=16, N=16) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_16X16_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_16X16_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=16) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X16_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X16_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (UINT32, M=32, N=32) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 5); -#if defined(__AVX2__) - POPCNT_UINT32_32X32_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX) -#else - POPCNT_UINT32_32X32_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE) -#endif -} -#endif // __SSSE3__ #if defined(AILEGO_M64) //! Compute the distance between matrix and query (UINT64, M=1, N=1) @@ -983,147 +359,6 @@ void HammingSquareRootDistanceMatrix::Compute( *out = std::sqrt(static_cast(HammingDistance(m, q, cnt))); } -#if defined(__AVX2__) -//! Compute the distance between matrix and query (UINT64, M=2, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_2X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=2, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_2X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=4, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_4X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=8, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_8X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=16, N=16) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_16X16_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=1) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=2) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=4) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=8) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=16) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X16_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} - -//! Compute the distance between matrix and query (UINT64, M=32, N=32) -void HammingSquareRootDistanceMatrix::Compute( - const ValueType *m, const ValueType *q, size_t dim, float *out) { - size_t cnt = (dim >> 6); - POPCNT_UINT64_32X32_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32) -} -#endif // __AVX2__ #endif // AILEGO_M64 } // namespace ailego diff --git a/src/ailego/math/hamming_distance_matrix.h b/src/ailego/math/hamming_distance_matrix.h index 5178e16..1ee67a5 100644 --- a/src/ailego/math/hamming_distance_matrix.h +++ b/src/ailego/math/hamming_distance_matrix.h @@ -83,248 +83,6 @@ struct HammingDistanceMatrix { float *out); }; -#if defined(__SSSE3__) -/*! Hamming Distance Matrix (UINT32, M=2, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=2, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=4, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=4, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=4, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=8, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=8, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=8, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=8, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=16, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=16, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=16, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=16, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=16, N=16) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=16) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT32, M=32, N=32) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // __SSSE3__ - #if defined(AILEGO_M64) /*! Hamming Distance Matrix (UINT64) */ @@ -381,247 +139,6 @@ struct HammingDistanceMatrix { float *out); }; -#if defined(__AVX2__) -/*! Hamming Distance Matrix (UINT64, M=2, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=2, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=4, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=4, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=4, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=8, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=8, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=8, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=8, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=16, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=16, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=16, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=16, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=16, N=16) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=1) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=2) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=4) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=8) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=16) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Distance Matrix (UINT64, M=32, N=32) - */ -template <> -struct HammingDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // __AVX2__ #endif // AILEGO_M64 /*! Hamming Square Root Distance Matrix @@ -656,247 +173,6 @@ struct HammingSquareRootDistanceMatrix { float *out); }; -#if defined(__SSSE3__) -/*! Hamming Square Root Distance Matrix (UINT32, M=2, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=2, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=16) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=16) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=32) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint32_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // __SSSE3__ #if defined(AILEGO_M64) /*! Hamming Square Root Distance Matrix (UINT64, M=1, N=1) @@ -911,247 +187,6 @@ struct HammingSquareRootDistanceMatrix { float *out); }; -#if defined(__AVX2__) -/*! Hamming Square Root Distance Matrix (UINT64, M=2, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=2, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=16) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=1) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=2) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=4) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=8) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=16) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=32) - */ -template <> -struct HammingSquareRootDistanceMatrix { - //! Type of value - using ValueType = uint64_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // __AVX2__ #endif // AILEGO_M64 } // namespace ailego diff --git a/src/ailego/math/inner_product_matrix.h b/src/ailego/math/inner_product_matrix.h index e1e8718..d141722 100644 --- a/src/ailego/math/inner_product_matrix.h +++ b/src/ailego/math/inner_product_matrix.h @@ -735,246 +735,6 @@ struct InnerProductMatrix { float *out); }; -/*! Inner Product Matrix (FP32, M=2, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=2, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=4, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=4, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=4, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=8, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=8, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=8, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=8, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=16, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=16, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=16, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=16, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=16, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP32, M=32, N=32) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Minus Inner Product Matrix (FP32, M=1, N=1) */ template <> @@ -986,246 +746,6 @@ struct MinusInnerProductMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -/*! Minus Inner Product Matrix (FP32, M=2, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=2, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=4, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=4, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=4, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=8, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=8, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=8, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=8, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=16, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=16, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=16, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=16, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=16, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP32, M=32, N=32) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = float; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; #endif // __SSE__ || __ARM_NEON #if (defined(__F16C__) && defined(__AVX__)) || \ @@ -1254,487 +774,6 @@ struct MinusInnerProductMatrix { float *out); }; -#if !defined(__ARM_NEON) -/*! Inner Product Matrix (FP16, M=2, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=2, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=4, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=4, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=4, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=8, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=8, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=8, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=8, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=16, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=16, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=16, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=16, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=16, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (FP16, M=32, N=32) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=2, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=2, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=4, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=4, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=4, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=8, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=8, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=8, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=8, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=16, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=16, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=16, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=16, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=16, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (FP16, M=32, N=32) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = Float16; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; -#endif // !__ARM_NEON #endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) #if defined(__SSE4_1__) @@ -1750,246 +789,6 @@ struct InnerProductMatrix { float *out); }; -/*! Inner Product Matrix (INT8, M=2, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=2, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=4, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=4, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=4, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=8, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=8, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=8, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=8, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=16, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=16, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=16, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=16, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=16, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT8, M=32, N=32) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Minus Inner Product Matrix (INT8, M=1, N=1) */ template <> @@ -2002,245 +801,6 @@ struct MinusInnerProductMatrix { float *out); }; -/*! Minus Inner Product Matrix (INT8, M=2, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=2, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=4, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=4, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=4, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=8, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=8, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=8, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=8, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=16, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=16, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=16, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=16, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=16, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT8, M=32, N=32) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = int8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; /*! Inner Product Matrix (INT4, M=1, N=1) */ @@ -2254,246 +814,6 @@ struct InnerProductMatrix { float *out); }; -/*! Inner Product Matrix (INT4, M=2, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=2, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=4, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=4, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=4, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=8, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=8, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=8, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=8, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=16, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=16, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=16, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=16, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=16, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=1) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=2) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=4) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=8) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=16) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Inner Product Matrix (INT4, M=32, N=32) - */ -template <> -struct InnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - /*! Minus Inner Product Matrix (INT4, M=1, N=1) */ template <> @@ -2505,246 +825,6 @@ struct MinusInnerProductMatrix { static void Compute(const ValueType *m, const ValueType *q, size_t dim, float *out); }; - -/*! Minus Inner Product Matrix (INT4, M=2, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=2, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=4, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=4, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=4, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=8, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=8, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=8, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=8, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=16, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=16, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=16, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=16, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=16, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=1) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=2) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=4) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=8) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=16) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; - -/*! Minus Inner Product Matrix (INT4, M=32, N=32) - */ -template <> -struct MinusInnerProductMatrix { - //! Type of value - using ValueType = uint8_t; - - //! Compute the distance between matrix and query - static void Compute(const ValueType *m, const ValueType *q, size_t dim, - float *out); -}; #endif // __SSE4_1__ template diff --git a/src/ailego/math/inner_product_matrix_fp16.cc b/src/ailego/math/inner_product_matrix_fp16.cc deleted file mode 100644 index 682cc91..0000000 --- a/src/ailego/math/inner_product_matrix_fp16.cc +++ /dev/null @@ -1,1948 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "distance_matrix_accum_fp16.i" -#include "inner_product_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_FP32_STEP_SSE FMA_FP32_SSE -#define ACCUM_FP32_STEP_AVX FMA_FP32_AVX -#define ACCUM_FP32_STEP_AVX512 FMA_FP32_AVX512 -#define ACCUM_FP32_STEP_NEON FMA_FP32_NEON -#define ACCUM_FP16_STEP_GENERAL FMA_FP16_GENERAL -#define ACCUM_FP16_STEP_NEON FMA_FP16_NEON - -#if defined(__AVX512F__) && !defined(__AVX512DQ__) -#define _mm512_xor_ps(a, b) \ - _mm512_castsi512_ps( \ - _mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b))) -#endif // __AVX512DQ__ - -#if defined(__SSE__) -static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f); -#endif // __SSE__ - -#if defined(__AVX__) -static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f); -#endif // __AVX__ - -#if defined(__AVX512F__) -static const __m512 NEGZEROS_FP32_AVX512 = _mm512_set1_ps(-0.0f); -#endif // __AVX512F__ - -//! Reverse sign of value (GENERAL) -#define NEGATE_FP32_GENERAL(v) -(v) - -#define NEGATE_FP32_SSE(v, ...) _mm_xor_ps(v, NEGZEROS_FP32_SSE) - -//! Reverse sign of value (AVX) -#define NEGATE_FP32_AVX(v, ...) _mm256_xor_ps(v, NEGZEROS_FP32_AVX) - -//! Reverse sign of value (AVX512) -#define NEGATE_FP32_AVX512(v, ...) _mm512_xor_ps(v, NEGZEROS_FP32_AVX512) - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ - xmm_sum = _mm_fmadd_ps(xmm_m, xmm_q, xmm_sum); - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ - ymm_sum = _mm256_fmadd_ps(ymm_m, ymm_q, ymm_sum); - -//! Calculate Fused-Multiply-Add (AVX512) -#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); - -//! Calculate Fused-Multiply-Add (AVX512FP16) -#define FMA_FP16_AVX512FP16(zmm_m, zmm_q, zmm_sum) \ - zmm_sum = _mm512_fmadd_ph(zmm_m, zmm_q, zmm_sum); - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_FP16_GENERAL(m, q, sum) sum += (m * q); - -//! Calculate Fused-Multiply-Add (NEON) -#define FMA_FP16_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f16(v_sum, v_m, v_q); - -//! Calculate Fused-Multiply-Add (NEON) -#define FMA_FP32_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_q); - -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) - -#if defined(__AVX512FP16__) -//! Inner Product -static inline float InnerProductAVX512FP16(const Float16 *lhs, - const Float16 *rhs, size_t size) { - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 6) << 6); - - __m512h zmm_sum_0 = _mm512_setzero_ph(); - __m512h zmm_sum_1 = _mm512_setzero_ph(); - - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0), - zmm_sum_0) - - FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32), - zmm_sum_1) - } - - if (last >= last_aligned + 32) { - FMA_FP16_AVX512FP16(_mm512_load_ph(lhs), _mm512_load_ph(rhs), zmm_sum_0) - lhs += 32; - rhs += 32; - } - } else { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0), - zmm_sum_0) - - FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32), - zmm_sum_1) - } - - if (last >= last_aligned + 32) { - FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs), zmm_sum_0) - lhs += 32; - rhs += 32; - } - } - - zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1); - - if (lhs != last) { - __mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1); - __m512i zmm_undefined = _mm512_undefined_epi32(); - zmm_sum_0 = _mm512_mask3_fmadd_ph( - _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)), - _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs)), - zmm_sum_0, mask); - } - - return HorizontalAdd_FP16_V512(zmm_sum_0); -} - -#endif - -//! Compute the distance between matrix and query (FP16, M=1, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, ) -#else -#if defined(__AVX512FP16__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { - *out = InnerProductAVX512FP16(m, q, dim); - return; - } -#endif //__AVX512FP16__ -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) - return; - } -#endif //__AVX512F__ - ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, ) -#endif //__ARM_NEON -} - -//! Compute the distance between matrix and query (FP16, M=1, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) -#else -#if defined(__AVX512FP16__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { - *out = -InnerProductAVX512FP16(m, q, dim); - return; - } -#endif //__AVX512FP16__ -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) - return; - } -#endif //__AVX512F__ - ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) -#endif //__ARM_NEON -} - -#if !defined(__ARM_NEON) -//! Compute the distance between matrix and query (FP16, M=2, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_2X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=2, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_2X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_4X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_4X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_4X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_8X1_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_8X2_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_8X4_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { - ACCUM_FP16_8X8_AVX(m, q, dim, out, ) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X1_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X1_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X2_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X2_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X4_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X4_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X8_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X8_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X16_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X16_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X1_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X1_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X2_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X2_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X4_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X4_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X8_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X8_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X16_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X16_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=32) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X32_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X32_AVX(m, q, dim, out, ) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=2, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -} - -//! Compute the distance between matrix and query (FP16, M=2, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -} - -//! Compute the distance between matrix and query (FP16, M=4, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -} - -//! Compute the distance between matrix and query (FP16, M=8, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { - ACCUM_FP16_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -} - -//! Compute the distance between matrix and query (FP16, M=16, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X1_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X2_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X4_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X8_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=16, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_16X16_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X1_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X2_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X4_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X8_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X16_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} - -//! Compute the distance between matrix and query (FP16, M=32, N=32) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX512F__) - ACCUM_FP16_32X32_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#else - ACCUM_FP16_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#endif // __AVX512F__ -} -#endif // !__ARM_NEON -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) - -// sparse -#if defined(__AVX__) -const static __m128i SHUFFLE_MASK256[256] = { - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, -127, -127), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 5, - 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, - 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, - 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 15, 14), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, - 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, - 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, - 4), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2), - _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), -}; - -constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; - -float InnerProductSparseInSegmentAVX(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const Float16 *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const Float16 *q_sparse_value) { - float sum = 0.0f; - - // handle if the first dim is zero - bool m_zero = false; - Float16 m_zero_value{0.0f}; - if (m_sparse_count > 0 && m_sparse_index[0] == 0) { - m_sparse_count--; - m_sparse_index++; - m_zero_value = *m_sparse_value++; - m_zero = true; - } - - bool q_zero = false; - Float16 q_zero_value{0.0f}; - if (q_sparse_count > 0 && q_sparse_index[0] == 0) { - q_sparse_count--; - q_sparse_index++; - q_zero_value = *q_sparse_value++; - q_zero = true; - } - - if (m_zero && q_zero) { - sum = m_zero_value * q_zero_value; - } - - size_t i1 = 0, i2 = 0; - size_t end1 = m_sparse_count / 8 * 8; - size_t end2 = q_sparse_count / 8 * 8; - - uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; - uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; - - Float16 *val_start_1 = reinterpret_cast(fixed_buffer_1); - Float16 *val_start_2 = reinterpret_cast(fixed_buffer_2); - - Float16 *val_1 = val_start_1; - Float16 *val_2 = val_start_2; - - if (i1 < end1 && i2 < end2) { - while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - } - - while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - } - - __m128i mm_index_m = - _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); - __m128i mm_index_q = - _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); - - while (true) { -#ifdef DEBUG_PRINT - std::cout << "index 1: " << std::endl; - print_data16(&mm_index_m); - - std::cout << "index 2: " << std::endl; - print_data16(&mm_index_q); -#endif - - __m128i mm_cmp_res = - _mm_cmpistrm(mm_index_q, mm_index_m, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - -#ifdef DEBUG_PRINT - std::cout << "cmp res: " << std::endl; - print_data16(&mm_cmp_res); -#endif - - int r = _mm_extract_epi32(mm_cmp_res, 0); - - if (r) { - int r1 = r; - - __m128i v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1])); - __m128i vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(val_1), vs); - val_1 += _mm_popcnt_u32(r1); - - mm_cmp_res = _mm_cmpistrm( - mm_index_m, mm_index_q, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - r = _mm_extract_epi32(mm_cmp_res, 0); - - r1 = r; - - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2])); - vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]); - - _mm_storeu_si128(reinterpret_cast<__m128i *>(val_2), vs); - val_2 += _mm_popcnt_u32(r1); - } - - const uint16_t id1_max = m_sparse_index[i1 + 7]; - - if (id1_max <= q_sparse_index[i2 + 7]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - mm_index_m = _mm_loadu_si128( - reinterpret_cast(&m_sparse_index[i1])); - } - - if (id1_max >= q_sparse_index[i2 + 7]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - mm_index_q = _mm_loadu_si128( - reinterpret_cast(&q_sparse_index[i2])); - } - } - } - -do_scalar: - while (i1 < m_sparse_count && i2 < q_sparse_count) { - if (m_sparse_index[i1] == q_sparse_index[i2]) { - *val_1++ = m_sparse_value[i1]; - *val_2++ = q_sparse_value[i2]; - - ++i1; - ++i2; - } else if (m_sparse_index[i1] < q_sparse_index[i2]) { - ++i1; - } else { - ++i2; - } - } - - size_t res_num = val_1 - val_start_1; - - size_t res_num8 = res_num / 8 * 8; - - if (res_num8) { - __m256 sum256 = _mm256_setzero_ps(); - - for (size_t k = 0; k < res_num8; k += 8) { - __m256 ymm_1 = - _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_1 + k))); - __m256 ymm_2 = - _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_2 + k))); - ACCUM_FP32_STEP_AVX(ymm_1, ymm_2, sum256); - } - - sum += HorizontalAdd_FP32_V256(sum256); - } - - for (size_t k = res_num8; k < res_num; ++k) - sum += val_start_1[k] * val_start_2[k]; - - return sum; -} - -#elif defined(__AVX512FP16__) -const static __m128i SHUFFLE_MASK256[256] = { - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, -127, -127), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 5, - 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, - 6, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, - 8, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, - 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, - 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, - 10, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 13, 12, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, - 12, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, 15, 14), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 5, 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 5, 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3, 2, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 9, 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, - 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, - 14, 13, 12, 11, 10), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, - 4, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, - 4, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, - 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 3, 2), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, - 8, 7, 6), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, - 2), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, - 4), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0), - _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2), - _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), -}; - -constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; - -float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const Float16 *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const Float16 *q_sparse_value) { - float sum = 0.0f; - - // handle if the first dim is zero - bool m_zero = false; - Float16 m_zero_value{0.0f}; - if (m_sparse_count > 0 && m_sparse_index[0] == 0) { - m_sparse_count--; - m_sparse_index++; - m_zero_value = *m_sparse_value++; - m_zero = true; - } - - bool q_zero = false; - Float16 q_zero_value{0.0f}; - if (q_sparse_count > 0 && q_sparse_index[0] == 0) { - q_sparse_count--; - q_sparse_index++; - q_zero_value = *q_sparse_value++; - q_zero = true; - } - - if (m_zero && q_zero) { - sum = m_zero_value * q_zero_value; - } - - size_t i1 = 0, i2 = 0; - size_t end1 = m_sparse_count / 8 * 8; - size_t end2 = q_sparse_count / 8 * 8; - - uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; - uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; - - Float16 *val_start_1 = reinterpret_cast(fixed_buffer_1); - Float16 *val_start_2 = reinterpret_cast(fixed_buffer_2); - - Float16 *val_1 = val_start_1; - Float16 *val_2 = val_start_2; - - if (i1 < end1 && i2 < end2) { - while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - } - - while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - } - - __m128i mm_index_m = - _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); - __m128i mm_index_q = - _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); - - while (true) { -#ifdef DEBUG_PRINT - std::cout << "index 1: " << std::endl; - print_data16(&mm_index_m); - - std::cout << "index 2: " << std::endl; - print_data16(&mm_index_q); -#endif - - __m128i mm_cmp_res = - _mm_cmpistrm(mm_index_q, mm_index_m, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - -#ifdef DEBUG_PRINT - std::cout << "cmp res: " << std::endl; - print_data16(&mm_cmp_res); -#endif - - int r = _mm_extract_epi32(mm_cmp_res, 0); - - if (r) { - int r1 = r; - - __m128i v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1])); - __m128h vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1])); - - _mm_storeu_ph(val_1, vs); - val_1 += _mm_popcnt_u32(r1); - - mm_cmp_res = _mm_cmpistrm( - mm_index_m, mm_index_q, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - r = _mm_extract_epi32(mm_cmp_res, 0); - - r1 = r; - - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2])); - vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1])); - - _mm_storeu_ph(val_2, vs); - val_2 += _mm_popcnt_u32(r1); - } - - const uint16_t id1_max = m_sparse_index[i1 + 7]; - - if (id1_max <= q_sparse_index[i2 + 7]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - mm_index_m = _mm_loadu_si128( - reinterpret_cast(&m_sparse_index[i1])); - } - - if (id1_max >= q_sparse_index[i2 + 7]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - mm_index_q = _mm_loadu_si128( - reinterpret_cast(&q_sparse_index[i2])); - } - } - } - -do_scalar: - while (i1 < m_sparse_count && i2 < q_sparse_count) { - if (m_sparse_index[i1] == q_sparse_index[i2]) { - *val_1++ = m_sparse_value[i1]; - *val_2++ = q_sparse_value[i2]; - - ++i1; - ++i2; - } else if (m_sparse_index[i1] < q_sparse_index[i2]) { - ++i1; - } else { - ++i2; - } - } - - size_t res_num = val_1 - val_start_1; - - size_t res_num8 = res_num / 8 * 8; - - if (res_num8) { - __m128h sum128 = _mm_set1_ph(0); - - for (size_t k = 0; k < res_num8; k += 8) { - sum128 = _mm_add_ph(sum128, _mm_mul_ph(_mm_loadu_ph(val_start_1 + k), - _mm_loadu_ph(val_start_2 + k))); - } - - Float16 __attribute__((aligned(16))) tmp_res[8]; - _mm_store_ph(tmp_res, sum128); - sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] + - tmp_res[5] + tmp_res[6] + tmp_res[7]); - } - - for (size_t k = res_num8; k < res_num; ++k) - sum += val_start_1[k] * val_start_2[k]; - - return sum; -} - -#else -float InnerProductSparseInSegment(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const Float16 *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const Float16 *q_sparse_value) { - float sum = 0.0f; - - size_t m_i = 0; - size_t q_i = 0; - while (m_i < m_sparse_count && q_i < q_sparse_count) { - if (m_sparse_index[m_i] == q_sparse_index[q_i]) { - sum += m_sparse_value[m_i] * q_sparse_value[q_i]; - - ++m_i; - ++q_i; - } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { - ++m_i; - } else { - ++q_i; - } - } - - return sum; -} -#endif // __AVX512FP16__ - -template <> -float MinusInnerProductSparseMatrix:: - ComputeInnerProductSparseInSegment(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const ValueType *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const ValueType *q_sparse_value) { -#if defined(__AVX__) - return InnerProductSparseInSegmentAVX(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#elif defined(__AVX512FP16__) - return InnerProductSparseInSegmentAVX512FP16(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#else - return InnerProductSparseInSegment(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#endif -} - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp16_avx.cc b/src/ailego/math/inner_product_matrix_fp16_avx.cc new file mode 100644 index 0000000..a68b1fb --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp16_avx.cc @@ -0,0 +1,706 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +// sparse +#if defined(__AVX__) +const static __m128i SHUFFLE_MASK256[256] = { + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, -127, -127), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 5, + 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, + 6, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, + 6, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7, + 6, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9, + 8, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11, + 10, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13, + 12, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10, + 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 15, 14), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10, + 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, + 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, + 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15, + 14, 13, 12, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, + 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, + 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, + 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, + 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, + 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, + 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, + 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + 4), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2), + _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), +}; + +constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; + +float InnerProductSparseInSegmentAVX(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const Float16 *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const Float16 *q_sparse_value) { + float sum = 0.0f; + + // handle if the first dim is zero + bool m_zero = false; + Float16 m_zero_value{0.0f}; + if (m_sparse_count > 0 && m_sparse_index[0] == 0) { + m_sparse_count--; + m_sparse_index++; + m_zero_value = *m_sparse_value++; + m_zero = true; + } + + bool q_zero = false; + Float16 q_zero_value{0.0f}; + if (q_sparse_count > 0 && q_sparse_index[0] == 0) { + q_sparse_count--; + q_sparse_index++; + q_zero_value = *q_sparse_value++; + q_zero = true; + } + + if (m_zero && q_zero) { + sum = m_zero_value * q_zero_value; + } + + size_t i1 = 0, i2 = 0; + size_t end1 = m_sparse_count / 8 * 8; + size_t end2 = q_sparse_count / 8 * 8; + + uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; + uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; + + Float16 *val_start_1 = reinterpret_cast(fixed_buffer_1); + Float16 *val_start_2 = reinterpret_cast(fixed_buffer_2); + + Float16 *val_1 = val_start_1; + Float16 *val_2 = val_start_2; + + if (i1 < end1 && i2 < end2) { + while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + } + + while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + } + + __m128i mm_index_m = + _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); + __m128i mm_index_q = + _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); + + while (true) { +#ifdef DEBUG_PRINT + std::cout << "index 1: " << std::endl; + print_data16(&mm_index_m); + + std::cout << "index 2: " << std::endl; + print_data16(&mm_index_q); +#endif + + __m128i mm_cmp_res = + _mm_cmpistrm(mm_index_q, mm_index_m, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + +#ifdef DEBUG_PRINT + std::cout << "cmp res: " << std::endl; + print_data16(&mm_cmp_res); +#endif + + int r = _mm_extract_epi32(mm_cmp_res, 0); + + if (r) { + int r1 = r; + + __m128i v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1])); + __m128i vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(val_1), vs); + val_1 += _mm_popcnt_u32(r1); + + mm_cmp_res = _mm_cmpistrm( + mm_index_m, mm_index_q, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + r = _mm_extract_epi32(mm_cmp_res, 0); + + r1 = r; + + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2])); + vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]); + + _mm_storeu_si128(reinterpret_cast<__m128i *>(val_2), vs); + val_2 += _mm_popcnt_u32(r1); + } + + const uint16_t id1_max = m_sparse_index[i1 + 7]; + + if (id1_max <= q_sparse_index[i2 + 7]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + mm_index_m = _mm_loadu_si128( + reinterpret_cast(&m_sparse_index[i1])); + } + + if (id1_max >= q_sparse_index[i2 + 7]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + mm_index_q = _mm_loadu_si128( + reinterpret_cast(&q_sparse_index[i2])); + } + } + } + +do_scalar: + while (i1 < m_sparse_count && i2 < q_sparse_count) { + if (m_sparse_index[i1] == q_sparse_index[i2]) { + *val_1++ = m_sparse_value[i1]; + *val_2++ = q_sparse_value[i2]; + + ++i1; + ++i2; + } else if (m_sparse_index[i1] < q_sparse_index[i2]) { + ++i1; + } else { + ++i2; + } + } + + size_t res_num = val_1 - val_start_1; + + size_t res_num8 = res_num / 8 * 8; + + if (res_num8) { + __m256 sum256 = _mm256_setzero_ps(); + + for (size_t k = 0; k < res_num8; k += 8) { + __m256 ymm_1 = + _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_1 + k))); + __m256 ymm_2 = + _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_2 + k))); + ACCUM_FP32_STEP_AVX(ymm_1, ymm_2, sum256); + } + + sum += HorizontalAdd_FP32_V256(sum256); + } + + for (size_t k = res_num8; k < res_num; ++k) + sum += val_start_1[k] * val_start_2[k]; + + return sum; +} + +#endif // __AVX__ + + +#if defined(__AVX__) +void InnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out) { + ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, ) +} + +void MinusInnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out) { + ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, NEGATE_FP32_GENERAL) +} +#endif +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp16_avx512.cc b/src/ailego/math/inner_product_matrix_fp16_avx512.cc new file mode 100644 index 0000000..7e07952 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp16_avx512.cc @@ -0,0 +1,766 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512FP16__) +//! Inner Product +float InnerProductAVX512FP16(const Float16 *lhs, const Float16 *rhs, + size_t size) { + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 6) << 6); + + __m512h zmm_sum_0 = _mm512_setzero_ph(); + __m512h zmm_sum_1 = _mm512_setzero_ph(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0), + zmm_sum_0) + + FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32), + zmm_sum_1) + } + + if (last >= last_aligned + 32) { + FMA_FP16_AVX512FP16(_mm512_load_ph(lhs), _mm512_load_ph(rhs), zmm_sum_0) + lhs += 32; + rhs += 32; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0), + zmm_sum_0) + + FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32), + zmm_sum_1) + } + + if (last >= last_aligned + 32) { + FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs), zmm_sum_0) + lhs += 32; + rhs += 32; + } + } + + zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1); + + if (lhs != last) { + __mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1); + __m512i zmm_undefined = _mm512_undefined_epi32(); + zmm_sum_0 = _mm512_mask3_fmadd_ph( + _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)), + _mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs)), + zmm_sum_0, mask); + } + + return HorizontalAdd_FP16_V512(zmm_sum_0); +} + +#endif + +// sparse +#if defined(__AVX512FP16__) +constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; + +float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const Float16 *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const Float16 *q_sparse_value) { + const static __m128i SHUFFLE_MASK256[256] = { + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, -127, -127), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 7, 6, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 7, 6, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 7, 6, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 9, 8, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 11, 10, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 13, 12, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 13, 12, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, + 10, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, + 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, + 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, + 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, 15, 14), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, + 10, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, + 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, + 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, + 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, + 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, + 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, + 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, + 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, + 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, + 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + 15, 14, 13, 12, 11, 10), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 11, 10, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 11, 10, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 11, 10, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 5, 4, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 5, 4, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 11, 10, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 7, 6, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 7, 6, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, + 12, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 9, 8, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 9, 8, 3, 2), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 9, 8, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, + 3, 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, + 9, 8, 7, 6), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 1, 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 3, 2), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, + 5, 4), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0), + _mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2), + _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + }; + + float sum = 0.0f; + + // handle if the first dim is zero + bool m_zero = false; + Float16 m_zero_value{0.0f}; + if (m_sparse_count > 0 && m_sparse_index[0] == 0) { + m_sparse_count--; + m_sparse_index++; + m_zero_value = *m_sparse_value++; + m_zero = true; + } + + bool q_zero = false; + Float16 q_zero_value{0.0f}; + if (q_sparse_count > 0 && q_sparse_index[0] == 0) { + q_sparse_count--; + q_sparse_index++; + q_zero_value = *q_sparse_value++; + q_zero = true; + } + + if (m_zero && q_zero) { + sum = m_zero_value * q_zero_value; + } + + size_t i1 = 0, i2 = 0; + size_t end1 = m_sparse_count / 8 * 8; + size_t end2 = q_sparse_count / 8 * 8; + + uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; + uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; + + Float16 *val_start_1 = reinterpret_cast(fixed_buffer_1); + Float16 *val_start_2 = reinterpret_cast(fixed_buffer_2); + + Float16 *val_1 = val_start_1; + Float16 *val_2 = val_start_2; + + if (i1 < end1 && i2 < end2) { + while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + } + + while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + } + + __m128i mm_index_m = + _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); + __m128i mm_index_q = + _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); + + while (true) { +#ifdef DEBUG_PRINT + std::cout << "index 1: " << std::endl; + print_data16(&mm_index_m); + + std::cout << "index 2: " << std::endl; + print_data16(&mm_index_q); +#endif + + __m128i mm_cmp_res = + _mm_cmpistrm(mm_index_q, mm_index_m, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + +#ifdef DEBUG_PRINT + std::cout << "cmp res: " << std::endl; + print_data16(&mm_cmp_res); +#endif + + int r = _mm_extract_epi32(mm_cmp_res, 0); + + if (r) { + int r1 = r; + + __m128i v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1])); + __m128h vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1])); + + _mm_storeu_ph(val_1, vs); + val_1 += _mm_popcnt_u32(r1); + + mm_cmp_res = _mm_cmpistrm( + mm_index_m, mm_index_q, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + r = _mm_extract_epi32(mm_cmp_res, 0); + + r1 = r; + + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2])); + vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1])); + + _mm_storeu_ph(val_2, vs); + val_2 += _mm_popcnt_u32(r1); + } + + const uint16_t id1_max = m_sparse_index[i1 + 7]; + + if (id1_max <= q_sparse_index[i2 + 7]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + mm_index_m = _mm_loadu_si128( + reinterpret_cast(&m_sparse_index[i1])); + } + + if (id1_max >= q_sparse_index[i2 + 7]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + mm_index_q = _mm_loadu_si128( + reinterpret_cast(&q_sparse_index[i2])); + } + } + } + +do_scalar: + while (i1 < m_sparse_count && i2 < q_sparse_count) { + if (m_sparse_index[i1] == q_sparse_index[i2]) { + *val_1++ = m_sparse_value[i1]; + *val_2++ = q_sparse_value[i2]; + + ++i1; + ++i2; + } else if (m_sparse_index[i1] < q_sparse_index[i2]) { + ++i1; + } else { + ++i2; + } + } + + size_t res_num = val_1 - val_start_1; + + size_t res_num8 = res_num / 8 * 8; + + if (res_num8) { + __m128h sum128 = _mm_set1_ph(0); + + for (size_t k = 0; k < res_num8; k += 8) { + sum128 = _mm_add_ph(sum128, _mm_mul_ph(_mm_loadu_ph(val_start_1 + k), + _mm_loadu_ph(val_start_2 + k))); + } + + Float16 __attribute__((aligned(16))) tmp_res[8]; + _mm_store_ph(tmp_res, sum128); + sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] + + tmp_res[5] + tmp_res[6] + tmp_res[7]); + } + + for (size_t k = res_num8; k < res_num; ++k) + sum += val_start_1[k] * val_start_2[k]; + + return sum; +} + +#endif // __AVX512FP16__ + +#if defined(__AVX512F__) +void InnerProductAVX512(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out) { + ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, ) +} + +void MinusInnerProductAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out) { + ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, NEGATE_FP32_GENERAL) +} +#endif //__AVX512F__ + + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp16_dispatch.cc b/src/ailego/math/inner_product_matrix_fp16_dispatch.cc new file mode 100644 index 0000000..8676013 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp16_dispatch.cc @@ -0,0 +1,162 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +float InnerProductNEON(const Float16 *lhs, const Float16 *rhs, size_t size); +float MinusInnerProductNEON(const Float16 *lhs, const Float16 *rhs, + size_t size); +#endif + +#if defined(__AVX__) +void InnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out); +void MinusInnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out); +float InnerProductSparseInSegmentAVX(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const Float16 *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const Float16 *q_sparse_value); +#endif + +#if defined(__AVX512F__) +void InnerProductAVX512(const Float16 *lhs, const Float16 *rhs, size_t size, + float *out); +void MinusInnerProductAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *out); +#endif + +#if defined(__AVX512FP16__) +float InnerProductAVX512FP16(const Float16 *lhs, const Float16 *rhs, + size_t size); +float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const Float16 *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const Float16 *q_sparse_value); +#endif + +#if (defined(__F16C__) && defined(__AVX__)) || \ + (defined(__ARM_NEON) && defined(__aarch64__)) +//! Compute the distance between matrix and query (FP16, M=1, N=1) +void InnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, size_t dim, + float *out) { +#if defined(__ARM_NEON) + *out = InnerProductNEON(m, q, dim); +#else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = InnerProductAVX512FP16(m, q, dim); + return; + } +#endif //__AVX512FP16__ +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + InnerProductAVX512(m, q, dim, out); + return; + } +#endif //__AVX512F__ + InnerProductAVX(m, q, dim, out); +#endif //__ARM_NEON +} + +//! Compute the distance between matrix and query (FP16, M=1, N=1) +void MinusInnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { +#if defined(__ARM_NEON) + *out = MinusInnerProductNEON(m, q, dim); +#else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = -InnerProductAVX512FP16(m, q, dim); + return; + } +#endif //__AVX512FP16__ +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + MinusInnerProductAVX512(m, q, dim, out); + return; + } +#endif //__AVX512F__ + + MinusInnerProductAVX(m, q, dim, out); + +#endif //__ARM_NEON +} + +#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) + +// sparse +float InnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const Float16 *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const Float16 *q_sparse_value) { + float sum = 0.0f; + + size_t m_i = 0; + size_t q_i = 0; + while (m_i < m_sparse_count && q_i < q_sparse_count) { + if (m_sparse_index[m_i] == q_sparse_index[q_i]) { + sum += m_sparse_value[m_i] * q_sparse_value[q_i]; + + ++m_i; + ++q_i; + } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { + ++m_i; + } else { + ++q_i; + } + } + + return sum; +} + +template <> +float MinusInnerProductSparseMatrix:: + ComputeInnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const ValueType *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const ValueType *q_sparse_value) { +#if defined(__AVX512FP16__) + return InnerProductSparseInSegmentAVX512FP16(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#elif defined(__AVX__) + return InnerProductSparseInSegmentAVX(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); + +#else + return InnerProductSparseInSegment(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#endif +} + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp16_neon.cc b/src/ailego/math/inner_product_matrix_fp16_neon.cc new file mode 100644 index 0000000..a7c3090 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp16_neon.cc @@ -0,0 +1,42 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +float InnerProductNEON(const Float16 *lhs, const Float16 *rhs, size_t size) { + float score; + + ACCUM_FP16_1X1_NEON(lhs, rhs, size, &score, 0ull, ) + + return score; +} + +float MinusInnerProductNEON(const Float16 *lhs, const Float16 *rhs, + size_t size) { + float score; + + ACCUM_FP16_1X1_NEON(lhs, rhs, size, &score, 0ull, NEGATE_FP32_GENERAL) + + return score; +} +#endif + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp32.cc b/src/ailego/math/inner_product_matrix_fp32.cc deleted file mode 100644 index 78e260d..0000000 --- a/src/ailego/math/inner_product_matrix_fp32.cc +++ /dev/null @@ -1,1180 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "distance_matrix_accum_fp32.i" -#include "inner_product_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_FP32_STEP_SSE FMA_FP32_SSE -#define ACCUM_FP32_STEP_AVX FMA_FP32_AVX -#define ACCUM_FP32_STEP_AVX512 FMA_FP32_AVX512 -#define ACCUM_FP32_STEP_NEON FMA_FP32_NEON - -#if defined(__AVX512F__) && !defined(__AVX512DQ__) -#define _mm512_xor_ps(a, b) \ - _mm512_castsi512_ps( \ - _mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b))) -#endif // __AVX512DQ__ - -#if defined(__SSE__) -static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f); -#endif // __SSE__ - -#if defined(__AVX__) -static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f); -#endif // __AVX__ - -#if defined(__AVX512F__) -static const __m512 NEGZEROS_FP32_AVX512 = _mm512_set1_ps(-0.0f); -#endif // __AVX512F__ - -//! Reverse sign of value (SSE) -#define NEGATE_FP32_SSE(v, ...) _mm_xor_ps(v, NEGZEROS_FP32_SSE) - -//! Reverse sign of value (AVX) -#define NEGATE_FP32_AVX(v, ...) _mm256_xor_ps(v, NEGZEROS_FP32_AVX) - -//! Reverse sign of value (AVX512) -#define NEGATE_FP32_AVX512(v, ...) _mm512_xor_ps(v, NEGZEROS_FP32_AVX512) - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_FP32_GENERAL(m, q, sum) sum += (m * q); - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_FP32_SSE(xmm_m, xmm_q, xmm_sum) \ - xmm_sum = _mm_fmadd_ps(xmm_m, xmm_q, xmm_sum); - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_FP32_AVX(ymm_m, ymm_q, ymm_sum) \ - ymm_sum = _mm256_fmadd_ps(ymm_m, ymm_q, ymm_sum); - -//! Calculate Fused-Multiply-Add (AVX512) -#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); - -//! Calculate Fused-Multiply-Add (NEON) -#define FMA_FP32_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_q); - -#if defined(__ARM_NEON) -//! Inner Product -static inline float InnerProductNEON(const float *lhs, const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - float32x4_t v_sum_0 = vdupq_n_f32(0); - float32x4_t v_sum_1 = vdupq_n_f32(0); - - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); - v_sum_1 = vfmaq_f32(v_sum_1, vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); - } - if (last >= last_aligned + 4) { - v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs), vld1q_f32(rhs)); - lhs += 4; - rhs += 4; - } - - float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); - switch (last - lhs) { - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __ARM_NEON - -#if defined(__SSE__) -//! Inner Product -static inline float InnerProductSSE(const float *lhs, const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - __m128 xmm_sum_0 = _mm_setzero_ps(); - __m128 xmm_sum_1 = _mm_setzero_ps(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); - __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); - __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); - __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); - xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); - xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); - } - - if (last >= last_aligned + 4) { - xmm_sum_0 = _mm_fmadd_ps(_mm_load_ps(lhs), _mm_load_ps(rhs), xmm_sum_0); - lhs += 4; - rhs += 4; - } - } else { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); - __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); - __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); - __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); - xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); - xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); - } - - if (last >= last_aligned + 4) { - xmm_sum_0 = _mm_fmadd_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs), xmm_sum_0); - lhs += 4; - rhs += 4; - } - } - float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); - - switch (last - lhs) { - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} - -#endif // __SSE__ - -// #if 1 -#if defined(__SSE4_1__) -const static __m128i SHUFFLE_MASK16[16] = { - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, -127, -127), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, - 4), - _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), -}; - -constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; - -float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const float *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const float *q_sparse_value) { - float sum = 0.0f; - - // handle if the first dim is zero - bool m_zero = false; - float m_zero_value = 0.0f; - if (m_sparse_count > 0 && m_sparse_index[0] == 0) { - m_sparse_count--; - m_sparse_index++; - m_zero_value = *m_sparse_value++; - m_zero = true; - } - - bool q_zero = false; - float q_zero_value = 0.0f; - if (q_sparse_count > 0 && q_sparse_index[0] == 0) { - q_sparse_count--; - q_sparse_index++; - q_zero_value = *q_sparse_value++; - q_zero = true; - } - - if (m_zero && q_zero) { - sum = m_zero_value * q_zero_value; - } - - size_t i1 = 0, i2 = 0; - size_t end1 = m_sparse_count / 8 * 8; - size_t end2 = q_sparse_count / 8 * 8; - - // std::vector mem1; - // std::vector mem2; - - float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; - float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; - - float *val_start_1 = fixed_buffer_1; - float *val_start_2 = fixed_buffer_2; - - // uint32_t max_count = std::max(m_sparse_count, q_sparse_count); - - // if (MAX_SPARSE_BUFFER_LENGTH < max_count) { - // mem1.reserve(max_count); - // mem2.reserve(max_count); - - // val_start_1 = mem1.data(); - // val_start_2 = mem2.data(); - // } - - float *val_1 = val_start_1; - float *val_2 = val_start_2; - - if (i1 < end1 && i2 < end2) { - while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - } - - while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - } - - __m128i mm_index_m = - _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); - __m128i mm_index_q = - _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); - - while (true) { -#ifdef DEBUG_PRINT - std::cout << "index 1: " << std::endl; - print_data16(&mm_index_m); - - std::cout << "index 2: " << std::endl; - print_data16(&mm_index_q); -#endif - - __m128i mm_cmp_res = - _mm_cmpistrm(mm_index_q, mm_index_m, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - -#ifdef DEBUG_PRINT - std::cout << "cmp res: " << std::endl; - print_data16(&mm_cmp_res); -#endif - - int r = _mm_extract_epi32(mm_cmp_res, 0); - - if (r) { - int r1 = r & 15; - - __m128i v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1])); - __m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); - - _mm_storeu_ps(val_1, vs); - val_1 += _mm_popcnt_u32(r1); - - int r2 = (r >> 4) & 15; - v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1 + 4])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); - _mm_storeu_ps(val_1, vs); - val_1 += _mm_popcnt_u32(r2); - - mm_cmp_res = _mm_cmpistrm( - mm_index_m, mm_index_q, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - r = _mm_extract_epi32(mm_cmp_res, 0); - - r1 = r & 15; - - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); - _mm_storeu_ps(val_2, vs); - val_2 += _mm_popcnt_u32(r1); - - r2 = (r >> 4) & 15; - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2 + 4])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); - _mm_storeu_ps(val_2, vs); - val_2 += _mm_popcnt_u32(r2); - } - - const uint16_t id1_max = m_sparse_index[i1 + 7]; - - if (id1_max <= q_sparse_index[i2 + 7]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - mm_index_m = _mm_loadu_si128( - reinterpret_cast(&m_sparse_index[i1])); - } - - if (id1_max >= q_sparse_index[i2 + 7]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - mm_index_q = _mm_loadu_si128( - reinterpret_cast(&q_sparse_index[i2])); - } - } - } - -do_scalar: - while (i1 < m_sparse_count && i2 < q_sparse_count) { - if (m_sparse_index[i1] == q_sparse_index[i2]) { - *val_1++ = m_sparse_value[i1]; - *val_2++ = q_sparse_value[i2]; - - ++i1; - ++i2; - } else if (m_sparse_index[i1] < q_sparse_index[i2]) { - ++i1; - } else { - ++i2; - } - } - - size_t res_num = val_1 - val_start_1; - - // if (res_num != val_2 - val_start_2) { - // std::cerr << "size mismatch!" << std::endl; - // } - - size_t res_num4 = res_num / 4 * 4; - - if (res_num4) { - __m128 sum128 = _mm_set1_ps(0); - - for (size_t k = 0; k < res_num4; k += 4) { - sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k), - _mm_loadu_ps(val_start_2 + k))); - } - - float __attribute__((aligned(16))) tmp_res[4]; - _mm_store_ps(tmp_res, sum128); - sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); - } - - for (size_t k = res_num4; k < res_num; ++k) - sum += val_start_1[k] * val_start_2[k]; - - return sum; -} -#else -float InnerProductSparseInSegment(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const float *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const float *q_sparse_value) { - float sum = 0.0f; - - size_t m_i = 0; - size_t q_i = 0; - while (m_i < m_sparse_count && q_i < q_sparse_count) { - if (m_sparse_index[m_i] == q_sparse_index[q_i]) { - sum += m_sparse_value[m_i] * q_sparse_value[q_i]; - - ++m_i; - ++q_i; - } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { - ++m_i; - } else { - ++q_i; - } - } - - return sum; -} -#endif // __SSE4_1__ - -template <> -float MinusInnerProductSparseMatrix::ComputeInnerProductSparseInSegment( - uint32_t m_sparse_count, const uint16_t *m_sparse_index, - const ValueType *m_sparse_value, uint32_t q_sparse_count, - const uint16_t *q_sparse_index, const ValueType *q_sparse_value) { -#if defined(__SSE4_1__) - return InnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#else - return InnerProductSparseInSegment(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#endif -} - -#if defined(__AVX__) -//! Inner Product -static inline float InnerProductAVX(const float *lhs, const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 4) << 4); - - __m256 ymm_sum_0 = _mm256_setzero_ps(); - __m256 ymm_sum_1 = _mm256_setzero_ps(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); - __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); - __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); - __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - } - - if (last >= last_aligned + 8) { - ymm_sum_0 = - _mm256_fmadd_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs), ymm_sum_0); - lhs += 8; - rhs += 8; - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); - __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); - __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); - __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - } - - if (last >= last_aligned + 8) { - ymm_sum_0 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs), - ymm_sum_0); - lhs += 8; - rhs += 8; - } - } - float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); - - switch (last - lhs) { - case 7: - FMA_FP32_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - FMA_FP32_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - FMA_FP32_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - FMA_FP32_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX__ - -#if defined(__AVX512F__) -//! Inner Product -static inline float InnerProductAVX512(const float *lhs, const float *rhs, - size_t size) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 5) << 5); - - __m512 zmm_sum_0 = _mm512_setzero_ps(); - __m512 zmm_sum_1 = _mm512_setzero_ps(); - - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - FMA_FP32_AVX512(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0), - zmm_sum_0) - - FMA_FP32_AVX512(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16), - zmm_sum_1) - } - - if (last >= last_aligned + 16) { - FMA_FP32_AVX512(_mm512_load_ps(lhs), _mm512_load_ps(rhs), zmm_sum_0) - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0), - zmm_sum_0) - - FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16), - zmm_sum_1) - } - - if (last >= last_aligned + 16) { - FMA_FP32_AVX512(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs), zmm_sum_0) - lhs += 16; - rhs += 16; - } - } - - zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); - if (lhs != last) { - __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); - __m512 zmm_undefined = _mm512_undefined_ps(); - zmm_sum_0 = _mm512_mask3_fmadd_ps( - _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), - _mm512_mask_loadu_ps(zmm_undefined, mask, rhs), zmm_sum_0, mask); - } - return HorizontalAdd_FP32_V512(zmm_sum_0); -} -#endif - -#if defined(__SSE__) || defined(__ARM_NEON) -//! Compute the distance between matrix and query (FP32, M=1, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - *out = InnerProductNEON(m, q, dim); -#else -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - if (dim > 15) { - *out = InnerProductAVX512(m, q, dim); - return; - } - } -#endif // __AVX512F__ -#if defined(__AVX__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { - if (dim > 7) { - *out = InnerProductAVX(m, q, dim); - return; - } - } -#endif // __AVX__ - *out = InnerProductSSE(m, q, dim); -#endif // __ARM_NEON -} - -//! Compute the distance between matrix and query (FP32, M=2, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_2X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_2X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=2, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_2X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_2X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X4_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_4X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_4X4_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X1_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X1_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X2_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X2_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X4_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X4_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X8_NEON(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_8X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_8X8_SSE(m, q, dim, out, ) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=16, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X1_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X1_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X1_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X2_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X2_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X2_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X4_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X4_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X4_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X8_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X8_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X8_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X16_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_16X16_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_16X16_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_16X16_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X1_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X1_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X1_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X1_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X2_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X2_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X2_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X2_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X4_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X4_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X4_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X4_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X8_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X8_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X8_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X8_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X16_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X16_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X16_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X16_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=32) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X32_NEON(m, q, dim, out, ) -#elif defined(__AVX512F__) - ACCUM_FP32_32X32_AVX512(m, q, dim, out, ) -#elif defined(__AVX__) - ACCUM_FP32_32X32_AVX(m, q, dim, out, ) -#else - ACCUM_FP32_32X32_SSE(m, q, dim, out, ) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=1, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - *out = -InnerProductNEON(m, q, dim); -#else -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - if (dim > 15) { - *out = -InnerProductAVX512(m, q, dim); - return; - } - } -#endif // __AVX512F__ -#if defined(__AVX__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { - if (dim > 7) { - *out = -InnerProductAVX(m, q, dim); - return; - } - } -#endif // __AVX__ - *out = -InnerProductSSE(m, q, dim); -#endif // __ARM_NEON -} - -//! Compute the distance between matrix and query (FP32, M=2, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X1_NEON(m, q, dim, out, vneg_f32) -#elif defined(__AVX__) - ACCUM_FP32_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_FP32_2X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=2, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_2X2_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_FP32_2X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X1_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_FP32_4X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X2_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_FP32_4X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=4, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_4X4_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_FP32_4X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X1_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_8X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X2_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_8X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X4_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_8X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=8, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_8X8_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX__) - ACCUM_FP32_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_8X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX__ -} - -//! Compute the distance between matrix and query (FP32, M=16, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X1_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X1_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_16X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X2_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X2_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_16X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X4_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X4_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_16X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X8_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X8_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_16X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=16, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_16X16_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_16X16_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_16X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X1_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X1_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X2_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X2_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X4_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X4_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X8_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X8_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X16_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X16_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} - -//! Compute the distance between matrix and query (FP32, M=32, N=32) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__ARM_NEON) - ACCUM_FP32_32X32_NEON(m, q, dim, out, vnegq_f32) -#elif defined(__AVX512F__) - ACCUM_FP32_32X32_AVX512(m, q, dim, out, NEGATE_FP32_AVX512) -#elif defined(__AVX__) - ACCUM_FP32_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_FP32_32X32_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif -} -#endif // __SSE__ || __ARM_NEON - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_fp32_avx.cc b/src/ailego/math/inner_product_matrix_fp32_avx.cc new file mode 100644 index 0000000..128adfd --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp32_avx.cc @@ -0,0 +1,94 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) +//! Inner Product +float InnerProductAVX(const float *lhs, const float *rhs, size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + ymm_sum_0 = + _mm256_fmadd_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs), ymm_sum_0); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + } + + if (last >= last_aligned + 8) { + ymm_sum_0 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs), + ymm_sum_0); + lhs += 8; + rhs += 8; + } + } + float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + + switch (last - lhs) { + case 7: + FMA_FP32_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_FP32_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_FP32_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_FP32_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __AVX__ + +} // namespace ailego +} // namespace zvec diff --git a/src/ailego/math/inner_product_matrix_fp32_avx512.cc b/src/ailego/math/inner_product_matrix_fp32_avx512.cc new file mode 100644 index 0000000..af3bf74 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp32_avx512.cc @@ -0,0 +1,75 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512F__) +//! Inner Product +float InnerProductAVX512(const float *lhs, const float *rhs, size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + FMA_FP32_AVX512(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0), + zmm_sum_0) + + FMA_FP32_AVX512(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16), + zmm_sum_1) + } + + if (last >= last_aligned + 16) { + FMA_FP32_AVX512(_mm512_load_ps(lhs), _mm512_load_ps(rhs), zmm_sum_0) + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0), + zmm_sum_0) + + FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16), + zmm_sum_1) + } + + if (last >= last_aligned + 16) { + FMA_FP32_AVX512(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs), zmm_sum_0) + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + zmm_sum_0 = _mm512_mask3_fmadd_ps( + _mm512_mask_loadu_ps(zmm_undefined, mask, lhs), + _mm512_mask_loadu_ps(zmm_undefined, mask, rhs), zmm_sum_0, mask); + } + return HorizontalAdd_FP32_V512(zmm_sum_0); +} + +#endif + +} // namespace ailego +} // namespace zvec diff --git a/src/ailego/math/inner_product_matrix_fp32_dispatch.cc b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc new file mode 100644 index 0000000..57acef2 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp32_dispatch.cc @@ -0,0 +1,97 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +float InnerProductNEON(const float *lhs, const float *rhs, size_t size); +float MinusInnerProductNEON(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__AVX512F__) +float InnerProductAVX512(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__AVX__) +float InnerProductAVX(const float *lhs, const float *rhs, size_t size); +float MinusInnerProductAVX(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__SSE__) +float InnerProductSSE(const float *lhs, const float *rhs, size_t size); +float MinusInnerProductSSE(const float *lhs, const float *rhs, size_t size); +#endif + +#if defined(__SSE__) || defined(__ARM_NEON) +//! Compute the distance between matrix and query (FP32, M=1, N=1) +void InnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, size_t dim, + float *out) { +#if defined(__ARM_NEON) + *out = InnerProductNEON(m, q, dim); +#else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = InnerProductAVX512(m, q, dim); + return; + } + } +#endif // __AVX512F__ +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = InnerProductAVX(m, q, dim); + return; + } + } +#endif // __AVX__ + *out = InnerProductSSE(m, q, dim); +#endif // __ARM_NEON +} + +//! Compute the distance between matrix and query (FP32, M=1, N=1) +void MinusInnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { +#if defined(__ARM_NEON) + *out = -InnerProductNEON(m, q, dim); +#else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = -InnerProductAVX512(m, q, dim); + return; + } + } +#endif // __AVX512F__ +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = -InnerProductAVX(m, q, dim); + return; + } + } +#endif // __AVX__ + *out = -InnerProductSSE(m, q, dim); +#endif // __ARM_NEON +} + +#endif +} // namespace ailego +} // namespace zvec diff --git a/src/ailego/math/inner_product_matrix_fp32_neon.cc b/src/ailego/math/inner_product_matrix_fp32_neon.cc new file mode 100644 index 0000000..e8626a3 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp32_neon.cc @@ -0,0 +1,57 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +//! Inner Product +float InnerProductNEON(const float *lhs, const float *rhs, size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs + 0), vld1q_f32(rhs + 0)); + v_sum_1 = vfmaq_f32(v_sum_1, vld1q_f32(lhs + 4), vld1q_f32(rhs + 4)); + } + if (last >= last_aligned + 4) { + v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs), vld1q_f32(rhs)); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + switch (last - lhs) { + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result) + } + return result; +} +#endif // __ARM_NEON + +} // namespace ailego +} // namespace zvec diff --git a/src/ailego/math/inner_product_matrix_fp32_sse.cc b/src/ailego/math/inner_product_matrix_fp32_sse.cc new file mode 100644 index 0000000..8a302bf --- /dev/null +++ b/src/ailego/math/inner_product_matrix_fp32_sse.cc @@ -0,0 +1,351 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE__) +//! Inner Product +float InnerProductSSE(const float *lhs, const float *rhs, size_t size) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum_0 = _mm_setzero_ps(); + __m128 xmm_sum_1 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); + xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + xmm_sum_0 = _mm_fmadd_ps(_mm_load_ps(lhs), _mm_load_ps(rhs), xmm_sum_0); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); + xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); + xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1); + } + + if (last >= last_aligned + 4) { + xmm_sum_0 = _mm_fmadd_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs), xmm_sum_0); + lhs += 4; + rhs += 4; + } + } + float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1)); + + switch (last - lhs) { + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +#endif // __SSE__ + +// #if 1 +#if defined(__SSE4_1__) +const static __m128i SHUFFLE_MASK16[16] = { + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, -127, -127), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + 4), + _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), +}; + +constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; + +float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value) { + float sum = 0.0f; + + // handle if the first dim is zero + bool m_zero = false; + float m_zero_value = 0.0f; + if (m_sparse_count > 0 && m_sparse_index[0] == 0) { + m_sparse_count--; + m_sparse_index++; + m_zero_value = *m_sparse_value++; + m_zero = true; + } + + bool q_zero = false; + float q_zero_value = 0.0f; + if (q_sparse_count > 0 && q_sparse_index[0] == 0) { + q_sparse_count--; + q_sparse_index++; + q_zero_value = *q_sparse_value++; + q_zero = true; + } + + if (m_zero && q_zero) { + sum = m_zero_value * q_zero_value; + } + + size_t i1 = 0, i2 = 0; + size_t end1 = m_sparse_count / 8 * 8; + size_t end2 = q_sparse_count / 8 * 8; + + // std::vector mem1; + // std::vector mem2; + + float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; + float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; + + float *val_start_1 = fixed_buffer_1; + float *val_start_2 = fixed_buffer_2; + + // uint32_t max_count = std::max(m_sparse_count, q_sparse_count); + + // if (MAX_SPARSE_BUFFER_LENGTH < max_count) { + // mem1.reserve(max_count); + // mem2.reserve(max_count); + + // val_start_1 = mem1.data(); + // val_start_2 = mem2.data(); + // } + + float *val_1 = val_start_1; + float *val_2 = val_start_2; + + if (i1 < end1 && i2 < end2) { + while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + } + + while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + } + + __m128i mm_index_m = + _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); + __m128i mm_index_q = + _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); + + while (true) { +#ifdef DEBUG_PRINT + std::cout << "index 1: " << std::endl; + print_data16(&mm_index_m); + + std::cout << "index 2: " << std::endl; + print_data16(&mm_index_q); +#endif + + __m128i mm_cmp_res = + _mm_cmpistrm(mm_index_q, mm_index_m, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + +#ifdef DEBUG_PRINT + std::cout << "cmp res: " << std::endl; + print_data16(&mm_cmp_res); +#endif + + int r = _mm_extract_epi32(mm_cmp_res, 0); + + if (r) { + int r1 = r & 15; + + __m128i v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1])); + __m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); + + _mm_storeu_ps(val_1, vs); + val_1 += _mm_popcnt_u32(r1); + + int r2 = (r >> 4) & 15; + v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1 + 4])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); + _mm_storeu_ps(val_1, vs); + val_1 += _mm_popcnt_u32(r2); + + mm_cmp_res = _mm_cmpistrm( + mm_index_m, mm_index_q, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + r = _mm_extract_epi32(mm_cmp_res, 0); + + r1 = r & 15; + + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); + _mm_storeu_ps(val_2, vs); + val_2 += _mm_popcnt_u32(r1); + + r2 = (r >> 4) & 15; + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2 + 4])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); + _mm_storeu_ps(val_2, vs); + val_2 += _mm_popcnt_u32(r2); + } + + const uint16_t id1_max = m_sparse_index[i1 + 7]; + + if (id1_max <= q_sparse_index[i2 + 7]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + mm_index_m = _mm_loadu_si128( + reinterpret_cast(&m_sparse_index[i1])); + } + + if (id1_max >= q_sparse_index[i2 + 7]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + mm_index_q = _mm_loadu_si128( + reinterpret_cast(&q_sparse_index[i2])); + } + } + } + +do_scalar: + while (i1 < m_sparse_count && i2 < q_sparse_count) { + if (m_sparse_index[i1] == q_sparse_index[i2]) { + *val_1++ = m_sparse_value[i1]; + *val_2++ = q_sparse_value[i2]; + + ++i1; + ++i2; + } else if (m_sparse_index[i1] < q_sparse_index[i2]) { + ++i1; + } else { + ++i2; + } + } + + size_t res_num = val_1 - val_start_1; + + // if (res_num != val_2 - val_start_2) { + // std::cerr << "size mismatch!" << std::endl; + // } + + size_t res_num4 = res_num / 4 * 4; + + if (res_num4) { + __m128 sum128 = _mm_set1_ps(0); + + for (size_t k = 0; k < res_num4; k += 4) { + sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k), + _mm_loadu_ps(val_start_2 + k))); + } + + float __attribute__((aligned(16))) tmp_res[4]; + _mm_store_ps(tmp_res, sum128); + sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); + } + + for (size_t k = res_num4; k < res_num; ++k) + sum += val_start_1[k] * val_start_2[k]; + + return sum; +} +#else +float InnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value) { + float sum = 0.0f; + + size_t m_i = 0; + size_t q_i = 0; + while (m_i < m_sparse_count && q_i < q_sparse_count) { + if (m_sparse_index[m_i] == q_sparse_index[q_i]) { + sum += m_sparse_value[m_i] * q_sparse_value[q_i]; + + ++m_i; + ++q_i; + } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { + ++m_i; + } else { + ++q_i; + } + } + + return sum; +} +#endif // __SSE4_1__ + +template <> +float MinusInnerProductSparseMatrix::ComputeInnerProductSparseInSegment( + uint32_t m_sparse_count, const uint16_t *m_sparse_index, + const ValueType *m_sparse_value, uint32_t q_sparse_count, + const uint16_t *q_sparse_index, const ValueType *q_sparse_value) { +#if defined(__SSE4_1__) + return InnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#else + return InnerProductSparseInSegment(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#endif +} + +} // namespace ailego +} // namespace zvec diff --git a/src/ailego/math/inner_product_matrix_int4.cc b/src/ailego/math/inner_product_matrix_int4.cc deleted file mode 100644 index 87a82e8..0000000 --- a/src/ailego/math/inner_product_matrix_int4.cc +++ /dev/null @@ -1,803 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include "distance_matrix_accum_int4.i" -#include "inner_product_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_INT4_STEP_SSE FMA_INT4_SSE -#define ACCUM_INT4_STEP_AVX FMA_INT4_AVX - -#if defined(__SSE4_1__) -//! Four-bits Convert Table -static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = { - 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1, - 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1}; -#endif // __SSE4_1__ - -#if defined(__SSE4_1__) -static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f); -static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f); -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -static const __m128i INT4_LOOKUP_SSE = - _mm_load_si128((const __m128i *)Int4ConvertTable); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f); -static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f); -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -static const __m256i INT4_LOOKUP_AVX = - _mm256_load_si256((const __m256i *)Int4ConvertTable); -#endif // __AVX2__ - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_INT4_GENERAL(m, q, sum) \ - sum += Int4MulTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \ - Int4MulTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)]; - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_INT4_SSE(xmm_m, xmm_q, xmm_sum) \ - { \ - __m128i xmm_lhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \ - _mm_and_si128((xmm_m), MASK_INT4_SSE)); \ - __m128i xmm_rhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \ - _mm_and_si128((xmm_q), MASK_INT4_SSE)); \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ - _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ - ONES_INT16_SSE), \ - xmm_sum); \ - xmm_lhs = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_m), 4), MASK_INT4_SSE)); \ - xmm_rhs = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_q), 4), MASK_INT4_SSE)); \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ - _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ - ONES_INT16_SSE), \ - xmm_sum); \ - } - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_INT4_AVX(ymm_m, ymm_q, ymm_sum) \ - { \ - __m256i ymm_lhs = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_m), MASK_INT4_AVX)); \ - __m256i ymm_rhs = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_q), MASK_INT4_AVX)); \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ - _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ - ONES_INT16_AVX), \ - ymm_sum); \ - ymm_lhs = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_m), 4), MASK_INT4_AVX)); \ - ymm_rhs = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_q), 4), MASK_INT4_AVX)); \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ - _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ - ONES_INT16_AVX), \ - ymm_sum); \ - } - -//! Compute the distance between matrix and query -#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ - { \ - __m128i xmm_lhs_0 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \ - __m128i xmm_rhs_0 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \ - __m128i xmm_lhs_1 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \ - __m128i xmm_rhs_1 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \ - xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); \ - xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); \ - xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); \ - xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); \ - xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), \ - ONES_INT16_SSE); \ - xmm_lhs_1 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), \ - ONES_INT16_SSE); \ - xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_lhs_1), xmm_sum); \ - } - -//! Compute the distance between matrix and query -#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ - { \ - __m256i ymm_lhs_0 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \ - __m256i ymm_rhs_0 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \ - __m256i ymm_lhs_1 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \ - __m256i ymm_rhs_1 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \ - ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); \ - ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); \ - ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); \ - ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); \ - ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), \ - ONES_INT16_AVX); \ - ymm_lhs_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), \ - ONES_INT16_AVX); \ - ymm_sum = \ - _mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_lhs_1), ymm_sum); \ - } - -//! Reverse sign of value (SSE) -#define NEGATE_FP32_SSE(v, ...) \ - _mm_xor_ps(_mm_cvtepi32_ps(v), NEGZEROS_FP32_SSE) - -//! Reverse sign of value (AVX) -#define NEGATE_FP32_AVX(v, ...) \ - _mm256_xor_ps(_mm256_cvtepi32_ps(v), NEGZEROS_FP32_AVX) - -//! Reverse sign of value (AVX512) -#define NEGATE_FP32_AVX512(v, ...) \ - _mm512_xor_ps(_mm512_cvtepi32_ps(v), NEGZEROS_FP32_AVX512) - -#if defined(__SSE4_1__) -//! Inner Product -static inline float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, - size_t size) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 4) << 4); - __m128i xmm_sum = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - } - } - float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); - - switch (last - lhs) { - case 15: - FMA_INT4_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - FMA_INT4_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - FMA_INT4_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - FMA_INT4_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - FMA_INT4_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - FMA_INT4_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - FMA_INT4_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - FMA_INT4_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - FMA_INT4_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - FMA_INT4_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - FMA_INT4_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - FMA_INT4_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - FMA_INT4_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_INT4_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_INT4_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __SSE4_1__ - -#if defined(__AVX2__) -//! Inner Product -static inline float InnerProductAVX(const uint8_t *lhs, const uint8_t *rhs, - size_t size) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 5) << 5); - __m256i ymm_sum = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); - FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), - ymm_sum); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); - FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) - ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), - ymm_sum); - lhs += 16; - rhs += 16; - } - } - float result = static_cast(HorizontalAdd_INT32_V256(ymm_sum)); - - switch (last - lhs) { - case 15: - FMA_INT4_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - FMA_INT4_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - FMA_INT4_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - FMA_INT4_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - FMA_INT4_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - FMA_INT4_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - FMA_INT4_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - FMA_INT4_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - FMA_INT4_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - FMA_INT4_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - FMA_INT4_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - FMA_INT4_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - FMA_INT4_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_INT4_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_INT4_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX2__ - -#if defined(__SSE4_1__) -//! Compute the distance between matrix and query (INT4, M=1, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - if (dim > 63) { - *out = InnerProductAVX(m, q, dim >> 1); - return; - } -#endif // __AVX2__ - *out = InnerProductSSE(m, q, dim >> 1); -} - -//! Compute the distance between matrix and query (INT4, M=2, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=2, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT4_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=32) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT4_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=1, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - if (dim > 63) { - *out = -InnerProductAVX(m, q, dim >> 1); - return; - } -#endif // __AVX2__ - *out = -InnerProductSSE(m, q, dim >> 1); -} - -//! Compute the distance between matrix and query (INT4, M=2, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT4_2X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=2, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT4_2X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT4_4X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT4_4X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=4, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT4_4X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_8X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_8X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_8X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=8, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_8X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_16X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_16X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_16X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_16X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=16, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_16X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT4, M=32, N=32) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT4_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT4_32X32_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int4_avx2.cc b/src/ailego/math/inner_product_matrix_int4_avx2.cc new file mode 100644 index 0000000..f69864a --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int4_avx2.cc @@ -0,0 +1,123 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int4.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Inner Product +float InnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, size_t size) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 5) << 5); + __m256i ymm_sum = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); + FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), + ymm_sum); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); + FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), + ymm_sum); + lhs += 16; + rhs += 16; + } + } + float result = static_cast(HorizontalAdd_INT32_V256(ymm_sum)); + + switch (last - lhs) { + case 15: + FMA_INT4_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + FMA_INT4_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + FMA_INT4_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + FMA_INT4_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + FMA_INT4_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + FMA_INT4_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + FMA_INT4_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + FMA_INT4_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + FMA_INT4_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_INT4_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_INT4_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_INT4_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_INT4_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_INT4_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_INT4_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +float MinusInnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, + size_t size) { + return -InnerProductAVX2(lhs, rhs, size); +} + +#endif // __AVX2__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int4_dispatch.cc b/src/ailego/math/inner_product_matrix_int4_dispatch.cc new file mode 100644 index 0000000..f26946d --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int4_dispatch.cc @@ -0,0 +1,62 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +float InnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, size_t size); +float MinusInnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, + size_t size); +#endif + +#if defined(__SSE4_1__) +float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size); +float MinusInnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size); +#endif + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query (INT4, M=1, N=1) +void InnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, size_t dim, + float *out) { +#if defined(__AVX2__) + if (dim > 63) { + *out = InnerProductAVX2(m, q, dim >> 1); + return; + } +#endif // __AVX2__ + *out = InnerProductSSE(m, q, dim >> 1); +} + +//! Compute the distance between matrix and query (INT4, M=1, N=1) +void MinusInnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { +#if defined(__AVX2__) + if (dim > 63) { + *out = MinusInnerProductAVX2(m, q, dim >> 1); + return; + } +#endif // __AVX2__ + *out = MinusInnerProductSSE(m, q, dim >> 1); +} + +#endif //__SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int4_sse.cc b/src/ailego/math/inner_product_matrix_int4_sse.cc new file mode 100644 index 0000000..11590bd --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int4_sse.cc @@ -0,0 +1,101 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int4.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Inner Product +float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 4) << 4); + __m128i xmm_sum = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) + } + } + float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); + + switch (last - lhs) { + case 15: + FMA_INT4_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + FMA_INT4_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + FMA_INT4_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + FMA_INT4_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + FMA_INT4_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + FMA_INT4_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + FMA_INT4_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + FMA_INT4_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + FMA_INT4_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_INT4_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_INT4_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_INT4_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_INT4_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_INT4_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_INT4_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +float MinusInnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, + size_t size) { + return -InnerProductSSE(lhs, rhs, size); +} + +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int8.cc b/src/ailego/math/inner_product_matrix_int8.cc deleted file mode 100644 index a307f69..0000000 --- a/src/ailego/math/inner_product_matrix_int8.cc +++ /dev/null @@ -1,841 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_int8.i" -#include "inner_product_matrix.h" - -namespace zvec { -namespace ailego { - -#define ACCUM_INT8_STEP_SSE FMA_INT8_SSE -#define ACCUM_INT8_STEP_AVX FMA_INT8_AVX - -#if defined(__AVX512F__) && !defined(__AVX512DQ__) -#define _mm512_xor_ps(a, b) \ - _mm512_castsi512_ps( \ - _mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b))) -#endif // __AVX512DQ__ - -#if defined(__SSE__) -static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f); -#endif // __SSE__ - -#if defined(__AVX__) -static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f); -#endif // __AVX__ - -#if defined(__AVX512F__) -static const __m512 NEGZEROS_FP32_AVX512 = _mm512_set1_ps(-0.0f); -#endif // __AVX512F__ - -#if defined(__SSE4_1__) -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -#endif // __AVX2__ - -//! Reverse sign of value (SSE) -#define NEGATE_FP32_SSE(v, ...) \ - _mm_xor_ps(_mm_cvtepi32_ps(v), NEGZEROS_FP32_SSE) - -//! Reverse sign of value (AVX) -#define NEGATE_FP32_AVX(v, ...) \ - _mm256_xor_ps(_mm256_cvtepi32_ps(v), NEGZEROS_FP32_AVX) - -//! Reverse sign of value (AVX512) -#define NEGATE_FP32_AVX512(v, ...) \ - _mm512_xor_ps(_mm512_cvtepi32_ps(v), NEGZEROS_FP32_AVX512) - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast(m * q); - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_INT8_SSE(xmm_m, xmm_q, xmm_sum) \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16( \ - _mm_maddubs_epi16(_mm_abs_epi8(xmm_q), _mm_sign_epi8(xmm_m, xmm_q)), \ - ONES_INT16_SSE), \ - xmm_sum); - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_INT8_AVX(ymm_m, ymm_q, ymm_sum) \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \ - _mm256_sign_epi8(ymm_m, ymm_q)), \ - ONES_INT16_AVX), \ - ymm_sum); - -#if defined(__SSE4_1__) -//! Inner Product -static inline float InnerProductSSE(const int8_t *lhs, const int8_t *rhs, - size_t size) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 5) << 5); - - __m128i xmm_sum_0 = _mm_setzero_si128(); - __m128i xmm_sum_1 = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); - - xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); - xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); - xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); - xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), - ONES_INT16_SSE), - xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), - ONES_INT16_SSE), - xmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - - xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); - xmm_rhs = _mm_abs_epi8(xmm_rhs); - xmm_sum_0 = _mm_add_epi32( - _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE), - xmm_sum_0); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); - - xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); - xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); - xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); - xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); - xmm_sum_0 = - _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), - ONES_INT16_SSE), - xmm_sum_0); - xmm_sum_1 = - _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), - ONES_INT16_SSE), - xmm_sum_1); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - - xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); - xmm_rhs = _mm_abs_epi8(xmm_rhs); - xmm_sum_0 = _mm_add_epi32( - _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE), - xmm_sum_0); - lhs += 16; - rhs += 16; - } - } - float result = static_cast( - HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1))); - - switch (last - lhs) { - case 15: - FMA_INT8_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - FMA_INT8_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - FMA_INT8_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - FMA_INT8_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - FMA_INT8_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - FMA_INT8_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - FMA_INT8_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - FMA_INT8_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - FMA_INT8_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - FMA_INT8_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - FMA_INT8_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - FMA_INT8_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - FMA_INT8_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_INT8_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_INT8_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __SSE4_1__ - -#if defined(__AVX2__) -//! Inner Product -static inline float InnerProductAVX(const int8_t *lhs, const int8_t *rhs, - size_t size) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 6) << 6); - float result = 0.0; - - __m256i ymm_sum_0 = _mm256_setzero_si256(); - __m256i ymm_sum_1 = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); - - ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); - ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); - ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); - ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); - - ymm_sum_0 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), - ONES_INT16_AVX), - ymm_sum_0); - ymm_sum_1 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), - ONES_INT16_AVX), - ymm_sum_1); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); - ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); - ymm_rhs = _mm256_abs_epi8(ymm_rhs); - ymm_sum_0 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), - ONES_INT16_AVX), - ymm_sum_0); - lhs += 32; - rhs += 32; - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); - xmm_rhs = _mm_abs_epi8(xmm_rhs); - ymm_sum_0 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), - _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), - ONES_INT16_SSE)), - ymm_sum_0); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); - - ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); - ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); - ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); - ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); - - ymm_sum_0 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), - ONES_INT16_AVX), - ymm_sum_0); - ymm_sum_1 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), - ONES_INT16_AVX), - ymm_sum_1); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); - ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); - ymm_rhs = _mm256_abs_epi8(ymm_rhs); - ymm_sum_0 = _mm256_add_epi32( - _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), - ONES_INT16_AVX), - ymm_sum_0); - lhs += 32; - rhs += 32; - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); - xmm_rhs = _mm_abs_epi8(xmm_rhs); - ymm_sum_0 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), - _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), - ONES_INT16_SSE)), - ymm_sum_0); - lhs += 16; - rhs += 16; - } - } - result = static_cast( - HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); - - switch (last - lhs) { - case 15: - FMA_INT8_GENERAL(lhs[14], rhs[14], result) - /* FALLTHRU */ - case 14: - FMA_INT8_GENERAL(lhs[13], rhs[13], result) - /* FALLTHRU */ - case 13: - FMA_INT8_GENERAL(lhs[12], rhs[12], result) - /* FALLTHRU */ - case 12: - FMA_INT8_GENERAL(lhs[11], rhs[11], result) - /* FALLTHRU */ - case 11: - FMA_INT8_GENERAL(lhs[10], rhs[10], result) - /* FALLTHRU */ - case 10: - FMA_INT8_GENERAL(lhs[9], rhs[9], result) - /* FALLTHRU */ - case 9: - FMA_INT8_GENERAL(lhs[8], rhs[8], result) - /* FALLTHRU */ - case 8: - FMA_INT8_GENERAL(lhs[7], rhs[7], result) - /* FALLTHRU */ - case 7: - FMA_INT8_GENERAL(lhs[6], rhs[6], result) - /* FALLTHRU */ - case 6: - FMA_INT8_GENERAL(lhs[5], rhs[5], result) - /* FALLTHRU */ - case 5: - FMA_INT8_GENERAL(lhs[4], rhs[4], result) - /* FALLTHRU */ - case 4: - FMA_INT8_GENERAL(lhs[3], rhs[3], result) - /* FALLTHRU */ - case 3: - FMA_INT8_GENERAL(lhs[2], rhs[2], result) - /* FALLTHRU */ - case 2: - FMA_INT8_GENERAL(lhs[1], rhs[1], result) - /* FALLTHRU */ - case 1: - FMA_INT8_GENERAL(lhs[0], rhs[0], result) - } - return result; -} -#endif // __AVX2__ - -#if defined(__SSE4_1__) -//! Compute the distance between matrix and query (INT8, M=1, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - if (dim > 31) { - *out = InnerProductAVX(m, q, dim); - return; - } -#endif // __AVX2__ - *out = InnerProductSSE(m, q, dim); -} - -//! Compute the distance between matrix and query (INT8, M=2, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=2, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps) -#else - ACCUM_INT8_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=1) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=2) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=4) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=8) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=16) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=32) -void InnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, size_t dim, - float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps) -#else - ACCUM_INT8_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=1, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - if (dim > 31) { - *out = -InnerProductAVX(m, q, dim); - return; - } -#endif // __AVX2__ - *out = -InnerProductSSE(m, q, dim); -} - -//! Compute the distance between matrix and query (INT8, M=2, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT8_2X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=2, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT8_2X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT8_4X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT8_4X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=4, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE) -#else - ACCUM_INT8_4X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_8X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_8X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_8X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=8, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_8X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_16X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_16X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_16X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_16X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=16, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_16X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=1) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X1_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=2) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X2_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=4) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X4_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=8) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X8_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=16) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X16_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} - -//! Compute the distance between matrix and query (INT8, M=32, N=32) -void MinusInnerProductMatrix::Compute(const ValueType *m, - const ValueType *q, - size_t dim, float *out) { -#if defined(__AVX2__) - ACCUM_INT8_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX) -#else - ACCUM_INT8_32X32_SSE(m, q, dim, out, NEGATE_FP32_SSE) -#endif // __AVX2__ -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int8_avx2.cc b/src/ailego/math/inner_product_matrix_int8_avx2.cc new file mode 100644 index 0000000..c32d698 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int8_avx2.cc @@ -0,0 +1,189 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Inner Product +float InnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 6) << 6); + float result = 0.0; + + __m256i ymm_sum_0 = _mm256_setzero_si256(); + __m256i ymm_sum_1 = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); + + ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); + ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); + ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); + ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); + + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), + ONES_INT16_AVX), + ymm_sum_0); + ymm_sum_1 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), + ONES_INT16_AVX), + ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); + ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); + ymm_rhs = _mm256_abs_epi8(ymm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), + ONES_INT16_AVX), + ymm_sum_0); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), + ONES_INT16_SSE)), + ymm_sum_0); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); + + ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); + ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); + ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); + ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); + + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), + ONES_INT16_AVX), + ymm_sum_0); + ymm_sum_1 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), + ONES_INT16_AVX), + ymm_sum_1); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); + ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs); + ymm_rhs = _mm256_abs_epi8(ymm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs), + ONES_INT16_AVX), + ymm_sum_0); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), + ONES_INT16_SSE)), + ymm_sum_0); + lhs += 16; + rhs += 16; + } + } + result = static_cast( + HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); + + switch (last - lhs) { + case 15: + FMA_INT8_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + FMA_INT8_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + FMA_INT8_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + FMA_INT8_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + FMA_INT8_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + FMA_INT8_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + FMA_INT8_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + FMA_INT8_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + FMA_INT8_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_INT8_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_INT8_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_INT8_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_INT8_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_INT8_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_INT8_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +float MinusInnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size) { + return -InnerProductAVX2(lhs, rhs, size); +} + +#endif // __AVX2__ + + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int8_dispatch.cc b/src/ailego/math/inner_product_matrix_int8_dispatch.cc new file mode 100644 index 0000000..5b75633 --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int8_dispatch.cc @@ -0,0 +1,60 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +float InnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size); +float MinusInnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size); +#endif + +#if defined(__SSE4_1__) +float InnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size); +float MinusInnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size); +#endif + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query (INT8, M=1, N=1) +void InnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, size_t dim, + float *out) { +#if defined(__AVX2__) + if (dim > 31) { + *out = InnerProductAVX2(m, q, dim); + return; + } +#endif // __AVX2__ + *out = InnerProductSSE(m, q, dim); +} + +//! Compute the distance between matrix and query (INT8, M=1, N=1) +void MinusInnerProductMatrix::Compute(const ValueType *m, + const ValueType *q, + size_t dim, float *out) { +#if defined(__AVX2__) + if (dim > 31) { + *out = MinusInnerProductAVX2(m, q, dim); + return; + } +#endif // __AVX2__ + *out = MinusInnerProductSSE(m, q, dim); +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/inner_product_matrix_int8_sse.cc b/src/ailego/math/inner_product_matrix_int8_sse.cc new file mode 100644 index 0000000..da0923c --- /dev/null +++ b/src/ailego/math/inner_product_matrix_int8_sse.cc @@ -0,0 +1,157 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_inner_product_utility.i" +#include "inner_product_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Inner Product +float InnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 5) << 5); + + __m128i xmm_sum_0 = _mm_setzero_si128(); + __m128i xmm_sum_1 = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); + + xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); + xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); + xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); + xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), + ONES_INT16_SSE), + xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), + ONES_INT16_SSE), + xmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + xmm_sum_0 = _mm_add_epi32( + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE), + xmm_sum_0); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); + + xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); + xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); + xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); + xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); + xmm_sum_0 = + _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), + ONES_INT16_SSE), + xmm_sum_0); + xmm_sum_1 = + _mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), + ONES_INT16_SSE), + xmm_sum_1); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + + xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs); + xmm_rhs = _mm_abs_epi8(xmm_rhs); + xmm_sum_0 = _mm_add_epi32( + _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE), + xmm_sum_0); + lhs += 16; + rhs += 16; + } + } + float result = static_cast( + HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1))); + + switch (last - lhs) { + case 15: + FMA_INT8_GENERAL(lhs[14], rhs[14], result) + /* FALLTHRU */ + case 14: + FMA_INT8_GENERAL(lhs[13], rhs[13], result) + /* FALLTHRU */ + case 13: + FMA_INT8_GENERAL(lhs[12], rhs[12], result) + /* FALLTHRU */ + case 12: + FMA_INT8_GENERAL(lhs[11], rhs[11], result) + /* FALLTHRU */ + case 11: + FMA_INT8_GENERAL(lhs[10], rhs[10], result) + /* FALLTHRU */ + case 10: + FMA_INT8_GENERAL(lhs[9], rhs[9], result) + /* FALLTHRU */ + case 9: + FMA_INT8_GENERAL(lhs[8], rhs[8], result) + /* FALLTHRU */ + case 8: + FMA_INT8_GENERAL(lhs[7], rhs[7], result) + /* FALLTHRU */ + case 7: + FMA_INT8_GENERAL(lhs[6], rhs[6], result) + /* FALLTHRU */ + case 6: + FMA_INT8_GENERAL(lhs[5], rhs[5], result) + /* FALLTHRU */ + case 5: + FMA_INT8_GENERAL(lhs[4], rhs[4], result) + /* FALLTHRU */ + case 4: + FMA_INT8_GENERAL(lhs[3], rhs[3], result) + /* FALLTHRU */ + case 3: + FMA_INT8_GENERAL(lhs[2], rhs[2], result) + /* FALLTHRU */ + case 2: + FMA_INT8_GENERAL(lhs[1], rhs[1], result) + /* FALLTHRU */ + case 1: + FMA_INT8_GENERAL(lhs[0], rhs[0], result) + } + return result; +} + +float MinusInnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size) { + return -InnerProductSSE(lhs, rhs, size); +} + +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16.cc deleted file mode 100644 index f4ede59..0000000 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp16.cc +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_fp16.i" -#include "mips_euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -//! Calculate Fused-Multiply-Add (AVX512) -#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); -#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \ - zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask); - -#define HorizontalAdd_FP16_NEON(v) \ - vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(v)), vcvt_high_f32_f16(v))) - -#define HorizontalAdd_FP32_V512_TO_V256(zmm) \ - _mm256_add_ps( \ - _mm512_castps512_ps256(zmm), \ - _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(zmm), 1))) - -//! Calculate Fused-Multiply-Add (AVX, FP16) -#define FMA_FP16_GENERAL(lhs, rhs, sum, norm1, norm2) \ - { \ - float v1 = lhs; \ - float v2 = rhs; \ - sum += v1 * v2; \ - norm1 += v1 * v1; \ - norm2 += v2 * v2; \ - } - -#if defined(__ARM_NEON) && defined(__aarch64__) -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormNEON(const Float16 *lhs, - const Float16 *rhs, - size_t size, float *sql, - float *sqr) { - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 3) << 3); - float16x8_t v_sum = vdupq_n_f16(0); - float16x8_t v_sum_norm1 = vdupq_n_f16(0); - float16x8_t v_sum_norm2 = vdupq_n_f16(0); - - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs); - float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs); - v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs); - v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs); - v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs); - } - if (last >= last_aligned + 4) { - float16x8_t v_lhs = vcombine_f16(vld1_f16((const float16_t *)lhs), - vreinterpret_f16_u64(vdup_n_u64(0ul))); - float16x8_t v_rhs = vcombine_f16(vld1_f16((const float16_t *)rhs), - vreinterpret_f16_u64(vdup_n_u64(0ul))); - v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs); - v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs); - v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs); - lhs += 4; - rhs += 4; - } - - float result = HorizontalAdd_FP16_NEON(v_sum); - float norm1 = HorizontalAdd_FP16_NEON(v_sum_norm1); - float norm2 = HorizontalAdd_FP16_NEON(v_sum_norm2); - - switch (last - lhs) { - case 3: - FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); - /* FALLTHRU */ - case 2: - FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); - /* FALLTHRU */ - case 1: - FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); - } - *sql = norm1; - *sqr = norm2; - return result; -} -#else -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormNEON(const Float16 *lhs, - const Float16 *rhs, - size_t size, float *sql, - float *sqr) { - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 3) << 3); - float32x4_t v_sum_0 = vdupq_n_f32(0); - float32x4_t v_sum_1 = vdupq_n_f32(0); - float32x4_t v_sum_norm1 = vdupq_n_f32(0); - float32x4_t v_sum_norm2 = vdupq_n_f32(0); - - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs); - float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs); - float32x4_t v_lhs_0 = vcvt_f32_f16(vget_low_f16(v_lhs)); - float32x4_t v_rhs_0 = vcvt_f32_f16(vget_low_f16(v_rhs)); - float32x4_t v_lhs_1 = vcvt_high_f32_f16(v_lhs); - float32x4_t v_rhs_1 = vcvt_high_f32_f16(v_rhs); - v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); - v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1); - } - if (last >= last_aligned + 4) { - float32x4_t v_lhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)lhs)); - float32x4_t v_rhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)rhs)); - v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); - lhs += 4; - rhs += 4; - } - - float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); - float norm1 = vaddvq_f32(v_sum_norm1); - float norm2 = vaddvq_f32(v_sum_norm2); - switch (last - lhs) { - case 3: - FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); - /* FALLTHRU */ - case 2: - FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); - /* FALLTHRU */ - case 1: - FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -#endif // __ARM_NEON && __aarch64__ - -#if defined(__AVX__) && defined(__F16C__) -#if defined(__AVX512F__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX512(const Float16 *lhs, - const Float16 *rhs, - size_t size, float *sql, - float *sqr) { - __m512 zmm_sum_0 = _mm512_setzero_ps(); - __m512 zmm_sum_1 = _mm512_setzero_ps(); - __m512 zmm_sum_norm1 = _mm512_setzero_ps(); - __m512 zmm_sum_norm2 = _mm512_setzero_ps(); - - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 5) << 5); - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512i zmm_lhs = _mm512_load_si512((const __m512i *)lhs); - __m512i zmm_rhs = _mm512_load_si512((const __m512i *)rhs); - __m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs)); - __m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1)); - __m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs)); - __m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1)); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) - } - if (last >= last_aligned + 16) { - __m512 zmm_lhs_0 = - _mm512_cvtph_ps(_mm256_load_si256((const __m256i *)lhs)); - __m512 zmm_rhs_0 = - _mm512_cvtph_ps(_mm256_load_si256((const __m256i *)rhs)); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512i zmm_lhs = _mm512_loadu_si512((const __m512i *)lhs); - __m512i zmm_rhs = _mm512_loadu_si512((const __m512i *)rhs); - __m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs)); - __m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1)); - __m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs)); - __m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1)); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) - } - if (last >= last_aligned + 16) { - __m512 zmm_lhs_0 = - _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)lhs)); - __m512 zmm_rhs_0 = - _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)rhs)); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - lhs += 16; - rhs += 16; - } - } - - __m256 ymm_sum_0 = - HorizontalAdd_FP32_V512_TO_V256(_mm512_add_ps(zmm_sum_0, zmm_sum_1)); - __m256 ymm_sum_norm1 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm1); - __m256 ymm_sum_norm2 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm2); - if (last >= lhs + 8) { - __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs)); - __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - lhs += 8; - rhs += 8; - } - - float result = HorizontalAdd_FP32_V256(ymm_sum_0); - float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); - float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); - switch (last - lhs) { - case 7: - FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2); - /* FALLTHRU */ - case 6: - FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2); - /* FALLTHRU */ - case 5: - FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2); - /* FALLTHRU */ - case 4: - FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2); - /* FALLTHRU */ - case 3: - FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); - /* FALLTHRU */ - case 2: - FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); - /* FALLTHRU */ - case 1: - FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); - } - - *sql = norm1; - *sqr = norm2; - return result; -} -#else -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX(const Float16 *lhs, - const Float16 *rhs, - size_t size, float *sql, - float *sqr) { - __m256 ymm_sum_0 = _mm256_setzero_ps(); - __m256 ymm_sum_1 = _mm256_setzero_ps(); - __m256 ymm_sum_norm1 = _mm256_setzero_ps(); - __m256 ymm_sum_norm2 = _mm256_setzero_ps(); - - const Float16 *last = lhs + size; - const Float16 *last_aligned = lhs + ((size >> 4) << 4); - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); - __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs)); - __m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1)); - __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs)); - __m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - if (last >= last_aligned + 8) { - __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)lhs)); - __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)rhs)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - lhs += 8; - rhs += 8; - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); - __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs)); - __m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1)); - __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs)); - __m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - if (last >= last_aligned + 8) { - __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs)); - __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs)); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - lhs += 8; - rhs += 8; - } - } - - float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); - float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); - float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); - switch (last - lhs) { - case 7: - FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2); - /* FALLTHRU */ - case 6: - FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2); - /* FALLTHRU */ - case 5: - FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2); - /* FALLTHRU */ - case 4: - FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2); - /* FALLTHRU */ - case 3: - FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); - /* FALLTHRU */ - case 2: - FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); - /* FALLTHRU */ - case 1: - FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); - } - - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __AVX512F__ -#endif // __AVX__ && __F16C__ - -#if (defined(__F16C__) && defined(__AVX__)) || \ - (defined(__ARM_NEON) && defined(__aarch64__)) -//! Compute the distance between matrix and query by SphericalInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { - float u2; - float v2; - float sum; - -#if defined(__ARM_NEON) - sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); -#elif defined(__AVX512F__) - sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); -#else - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); -#endif - - *out = ComputeSphericalInjection(sum, u2, v2, e2); -} - -//! Compute the distance between matrix and query by RepeatedQuadraticInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, - float *out) { - float u2; - float v2; - float sum; - -#if defined(__ARM_NEON) - sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); -#elif defined(__AVX512F__) - sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); -#else - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); -#endif - - sum = e2 * (u2 + v2 - 2 * sum); - u2 *= e2; - v2 *= e2; - for (size_t i = 0; i < m; ++i) { - sum += (u2 - v2) * (u2 - v2); - u2 = u2 * u2; - v2 = v2 * v2; - } - *out = sum; -} -#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx.cc new file mode 100644 index 0000000..c93edc1 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx.cc @@ -0,0 +1,116 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) && defined(__F16C__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr) { + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm2 = _mm256_setzero_ps(); + + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 4) << 4); + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); + __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs)); + __m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1)); + __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs)); + __m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)lhs)); + __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); + __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs)); + __m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1)); + __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs)); + __m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs)); + __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } + + float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); + float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); + switch (last - lhs) { + case 7: + FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2); + /* FALLTHRU */ + case 6: + FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2); + /* FALLTHRU */ + case 5: + FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2); + /* FALLTHRU */ + case 4: + FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2); + /* FALLTHRU */ + case 3: + FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); + /* FALLTHRU */ + case 2: + FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); + /* FALLTHRU */ + case 1: + FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); + } + + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __AVX__ && __F16C__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx512.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx512.cc new file mode 100644 index 0000000..51ce4fc --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_avx512.cc @@ -0,0 +1,134 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512F__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr) { + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm2 = _mm512_setzero_ps(); + + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 5) << 5); + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512i zmm_lhs = _mm512_load_si512((const __m512i *)lhs); + __m512i zmm_rhs = _mm512_load_si512((const __m512i *)rhs); + __m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs)); + __m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1)); + __m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs)); + __m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1)); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = + _mm512_cvtph_ps(_mm256_load_si256((const __m256i *)lhs)); + __m512 zmm_rhs_0 = + _mm512_cvtph_ps(_mm256_load_si256((const __m256i *)rhs)); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512i zmm_lhs = _mm512_loadu_si512((const __m512i *)lhs); + __m512i zmm_rhs = _mm512_loadu_si512((const __m512i *)rhs); + __m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs)); + __m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1)); + __m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs)); + __m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1)); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = + _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)lhs)); + __m512 zmm_rhs_0 = + _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)rhs)); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } + + __m256 ymm_sum_0 = + HorizontalAdd_FP32_V512_TO_V256(_mm512_add_ps(zmm_sum_0, zmm_sum_1)); + __m256 ymm_sum_norm1 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm1); + __m256 ymm_sum_norm2 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm2); + if (last >= lhs + 8) { + __m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs)); + __m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs)); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + + float result = HorizontalAdd_FP32_V256(ymm_sum_0); + float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); + float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); + switch (last - lhs) { + case 7: + FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2); + /* FALLTHRU */ + case 6: + FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2); + /* FALLTHRU */ + case 5: + FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2); + /* FALLTHRU */ + case 4: + FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2); + /* FALLTHRU */ + case 3: + FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); + /* FALLTHRU */ + case 2: + FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); + /* FALLTHRU */ + case 1: + FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); + } + + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __AVX512F__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_dispatch.cc new file mode 100644 index 0000000..b99ab45 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_dispatch.cc @@ -0,0 +1,96 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__AVX512F__) +float InnerProductAndSquaredNormAVX512(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__AVX__) +float InnerProductAndSquaredNormAVX(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if (defined(__F16C__) && defined(__AVX__)) || \ + (defined(__ARM_NEON) && defined(__aarch64__)) +//! Compute the distance between matrix and query by SphericalInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__ARM_NEON) + sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); +#else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); + } else +#endif //__AVX512F__ + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); + } +#endif //__ARM_NEON + + *out = ComputeSphericalInjection(sum, u2, v2, e2); +} + +//! Compute the distance between matrix and query by RepeatedQuadraticInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, + float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__ARM_NEON) + sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); +#else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); + } else +#endif //__AVX512F__ + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); + } +#endif //__ARM_NEON + + sum = e2 * (u2 + v2 - 2 * sum); + u2 *= e2; + v2 *= e2; + for (size_t i = 0; i < m; ++i) { + sum += (u2 - v2) * (u2 - v2); + u2 = u2 * u2; + v2 = v2 * v2; + } + *out = sum; +} + +#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc new file mode 100644 index 0000000..22493b4 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp16_neon.cc @@ -0,0 +1,126 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp16.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) && defined(__aarch64__) +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr) { + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 3) << 3); + float16x8_t v_sum = vdupq_n_f16(0); + float16x8_t v_sum_norm1 = vdupq_n_f16(0); + float16x8_t v_sum_norm2 = vdupq_n_f16(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs); + float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs); + v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs); + v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs); + v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs); + } + if (last >= last_aligned + 4) { + float16x8_t v_lhs = vcombine_f16(vld1_f16((const float16_t *)lhs), + vreinterpret_f16_u64(vdup_n_u64(0ul))); + float16x8_t v_rhs = vcombine_f16(vld1_f16((const float16_t *)rhs), + vreinterpret_f16_u64(vdup_n_u64(0ul))); + v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs); + v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs); + v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs); + lhs += 4; + rhs += 4; + } + + float result = HorizontalAdd_FP16_NEON(v_sum); + float norm1 = HorizontalAdd_FP16_NEON(v_sum_norm1); + float norm2 = HorizontalAdd_FP16_NEON(v_sum_norm2); + + switch (last - lhs) { + case 3: + FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); + /* FALLTHRU */ + case 2: + FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); + /* FALLTHRU */ + case 1: + FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); + } + *sql = norm1; + *sqr = norm2; + return result; +} +#else +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs, + size_t size, float *sql, float *sqr) { + const Float16 *last = lhs + size; + const Float16 *last_aligned = lhs + ((size >> 3) << 3); + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + float32x4_t v_sum_norm1 = vdupq_n_f32(0); + float32x4_t v_sum_norm2 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs); + float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs); + float32x4_t v_lhs_0 = vcvt_f32_f16(vget_low_f16(v_lhs)); + float32x4_t v_rhs_0 = vcvt_f32_f16(vget_low_f16(v_rhs)); + float32x4_t v_lhs_1 = vcvt_high_f32_f16(v_lhs); + float32x4_t v_rhs_1 = vcvt_high_f32_f16(v_rhs); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1); + } + if (last >= last_aligned + 4) { + float32x4_t v_lhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)lhs)); + float32x4_t v_rhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)rhs)); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + float norm1 = vaddvq_f32(v_sum_norm1); + float norm2 = vaddvq_f32(v_sum_norm2); + switch (last - lhs) { + case 3: + FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2); + /* FALLTHRU */ + case 2: + FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2); + /* FALLTHRU */ + case 1: + FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2); + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +#endif // __ARM_NEON && __aarch64__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc deleted file mode 100644 index f14117a..0000000 --- a/src/ailego/math/mips_euclidean_distance_matrix_fp32.cc +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_fp32.i" -#include "mips_euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_FP32_GENERAL(lhs, rhs, sum, norm1, norm2) \ - { \ - sum += (lhs) * (rhs); \ - norm1 += (lhs) * (lhs); \ - norm2 += (rhs) * (rhs); \ - } - -//! Calculate Fused-Multiply-Add (AVX512) -#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \ - zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum); -#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \ - zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask); - -#if defined(__ARM_NEON) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormNEON(const float *lhs, - const float *rhs, - size_t size, float *sql, - float *sqr) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - float32x4_t v_sum_0 = vdupq_n_f32(0); - float32x4_t v_sum_1 = vdupq_n_f32(0); - float32x4_t v_sum_norm1 = vdupq_n_f32(0); - float32x4_t v_sum_norm2 = vdupq_n_f32(0); - - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - float32x4_t v_lhs_0 = vld1q_f32(lhs + 0); - float32x4_t v_lhs_1 = vld1q_f32(lhs + 4); - float32x4_t v_rhs_0 = vld1q_f32(rhs + 0); - float32x4_t v_rhs_1 = vld1q_f32(rhs + 4); - v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); - v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1); - } - if (last >= last_aligned + 4) { - float32x4_t v_lhs_0 = vld1q_f32(lhs); - float32x4_t v_rhs_0 = vld1q_f32(rhs); - v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); - v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); - v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); - lhs += 4; - rhs += 4; - } - - float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); - float norm1 = vaddvq_f32(v_sum_norm1); - float norm2 = vaddvq_f32(v_sum_norm2); - switch (last - lhs) { - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __ARM_NEON - -#if defined(__SSE__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormSSE(const float *lhs, - const float *rhs, size_t size, - float *sql, float *sqr) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 3) << 3); - - __m128 xmm_sum = _mm_setzero_ps(); - __m128 xmm_sum_norm1 = _mm_setzero_ps(); - __m128 xmm_sum_norm2 = _mm_setzero_ps(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); - __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); - __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); - __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); - xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); - xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); - } - - if (last >= last_aligned + 4) { - __m128 xmm_lhs_0 = _mm_load_ps(lhs); - __m128 xmm_rhs_0 = _mm_load_ps(rhs); - xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - lhs += 4; - rhs += 4; - } - } else { - for (; lhs != last_aligned; lhs += 8, rhs += 8) { - __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); - __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); - __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); - __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); - xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); - xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); - } - - if (last >= last_aligned + 4) { - __m128 xmm_lhs_0 = _mm_loadu_ps(lhs); - __m128 xmm_rhs_0 = _mm_loadu_ps(rhs); - xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); - xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - lhs += 4; - rhs += 4; - } - } - float result = HorizontalAdd_FP32_V128(xmm_sum); - float norm1 = HorizontalAdd_FP32_V128(xmm_sum_norm1); - float norm2 = HorizontalAdd_FP32_V128(xmm_sum_norm2); - - switch (last - lhs) { - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __SSE__ - -#if defined(__AVX__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX(const float *lhs, - const float *rhs, size_t size, - float *sql, float *sqr) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 4) << 4); - - __m256 ymm_sum_0 = _mm256_setzero_ps(); - __m256 ymm_sum_1 = _mm256_setzero_ps(); - __m256 ymm_sum_norm1 = _mm256_setzero_ps(); - __m256 ymm_sum_norm2 = _mm256_setzero_ps(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); - __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); - __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); - __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - - if (last >= last_aligned + 8) { - __m256 ymm_lhs_0 = _mm256_load_ps(lhs); - __m256 ymm_rhs_0 = _mm256_load_ps(rhs); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - lhs += 8; - rhs += 8; - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); - __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); - __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); - __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - - if (last >= last_aligned + 8) { - __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs); - __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs); - ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - lhs += 8; - rhs += 8; - } - } - float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); - float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); - float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); - - switch (last - lhs) { - case 7: - FMA_FP32_GENERAL(lhs[6], rhs[6], result, norm1, norm2) - /* FALLTHRU */ - case 6: - FMA_FP32_GENERAL(lhs[5], rhs[5], result, norm1, norm2) - /* FALLTHRU */ - case 5: - FMA_FP32_GENERAL(lhs[4], rhs[4], result, norm1, norm2) - /* FALLTHRU */ - case 4: - FMA_FP32_GENERAL(lhs[3], rhs[3], result, norm1, norm2) - /* FALLTHRU */ - case 3: - FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __AVX__ - -#if defined(__AVX512F__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX512(const float *lhs, - const float *rhs, - size_t size, float *sql, - float *sqr) { - const float *last = lhs + size; - const float *last_aligned = lhs + ((size >> 5) << 5); - - __m512 zmm_sum_0 = _mm512_setzero_ps(); - __m512 zmm_sum_1 = _mm512_setzero_ps(); - __m512 zmm_sum_norm1 = _mm512_setzero_ps(); - __m512 zmm_sum_norm2 = _mm512_setzero_ps(); - - if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512 zmm_lhs_0 = _mm512_load_ps(lhs + 0); - __m512 zmm_lhs_1 = _mm512_load_ps(lhs + 16); - __m512 zmm_rhs_0 = _mm512_load_ps(rhs + 0); - __m512 zmm_rhs_1 = _mm512_load_ps(rhs + 16); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) - } - - if (last >= last_aligned + 16) { - __m512 zmm_lhs_0 = _mm512_load_ps(lhs); - __m512 zmm_rhs_0 = _mm512_load_ps(rhs); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs + 0); - __m512 zmm_lhs_1 = _mm512_loadu_ps(lhs + 16); - __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs + 0); - __m512 zmm_rhs_1 = _mm512_loadu_ps(rhs + 16); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) - } - - if (last >= last_aligned + 16) { - __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs); - __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs); - FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) - FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) - FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) - lhs += 16; - rhs += 16; - } - } - - zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); - if (lhs != last) { - __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); - __m512 zmm_undefined = _mm512_undefined_ps(); - __m512 zmm_lhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, lhs); - __m512 zmm_rhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, rhs); - FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0, mask); - FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1, mask); - FMA_MASK_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2, mask); - } - - *sql = HorizontalAdd_FP32_V512(zmm_sum_norm1); - *sqr = HorizontalAdd_FP32_V512(zmm_sum_norm2); - return HorizontalAdd_FP32_V512(zmm_sum_0); -} -#endif // __AVX512F__ - -#if defined(__SSE__) -//! Compute the distance between matrix and query by SphericalInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX512F__) - if (dim > 15) { - sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); - } else -#endif // __AVX512F__ -#if defined(__AVX__) - if (dim > 7) { - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); - } else -#endif // __AVX__ - { - sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); - } - - *out = ComputeSphericalInjection(sum, u2, v2, e2); -} - -//! Compute the distance between matrix and query by RepeatedQuadraticInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, - float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX512F__) - if (dim > 15) { - sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); - } else -#endif // __AVX512F__ -#if defined(__AVX__) - if (dim > 7) { - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); - } else -#endif // __AVX__ - { - sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); - } - - sum = e2 * (u2 + v2 - 2 * sum); - u2 *= e2; - v2 *= e2; - for (size_t i = 0; i < m; ++i) { - sum += (u2 - v2) * (u2 - v2); - u2 = u2 * u2; - v2 = v2 * v2; - } - *out = sum; -} -#endif // __SSE__ - -// #if 1 -#if defined(__SSE4_1__) -const static __m128i SHUFFLE_MASK16[16] = { - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, -127, -127, -127, -127), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, - 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, - -127, -127, 15, 14, 13, 12), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 3, 2, 1, 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 7, 6, 5, 4), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, - 11, 10, 9, 8), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, - 0), - _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, - 4), - _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), -}; - -constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; - -float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const float *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const float *q_sparse_value) { - float sum = 0.0f; - - // size_t alloc_size = 0; - - size_t i1 = 0, i2 = 0; - size_t end1 = m_sparse_count / 8 * 8; - size_t end2 = q_sparse_count / 8 * 8; - - // std::vector mem1; - // std::vector mem2; - - float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; - float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; - - float *val_start_1 = fixed_buffer_1; - float *val_start_2 = fixed_buffer_2; - - // uint32_t max_count = std::max(m_sparse_count, q_sparse_count); - - // if (MAX_SPARSE_BUFFER_LENGTH < max_count) { - // mem1.reserve(max_count); - // mem2.reserve(max_count); - - // val_start_1 = mem1.data(); - // val_start_2 = mem2.data(); - // } - - float *val_1 = val_start_1; - float *val_2 = val_start_2; - - if (i1 < end1 && i2 < end2) { - while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - } - - while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - } - - __m128i mm_index_m = - _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); - __m128i mm_index_q = - _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); - - while (true) { -#ifdef DEBUG_PRINT - std::cout << "index 1: " << std::endl; - print_data16(&mm_index_m); - - std::cout << "index 2: " << std::endl; - print_data16(&mm_index_q); -#endif - - __m128i mm_cmp_res = - _mm_cmpistrm(mm_index_q, mm_index_m, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - -#ifdef DEBUG_PRINT - std::cout << "cmp res: " << std::endl; - print_data16(&mm_cmp_res); -#endif - - int r = _mm_extract_epi32(mm_cmp_res, 0); - - if (r) { - int r1 = r & 15; - - __m128i v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1])); - __m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); - - _mm_storeu_ps(val_1, vs); - val_1 += _mm_popcnt_u32(r1); - - int r2 = (r >> 4) & 15; - v = _mm_loadu_si128( - reinterpret_cast(&m_sparse_value[i1 + 4])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); - _mm_storeu_ps(val_1, vs); - val_1 += _mm_popcnt_u32(r2); - - mm_cmp_res = _mm_cmpistrm( - mm_index_m, mm_index_q, - _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); - r = _mm_extract_epi32(mm_cmp_res, 0); - - r1 = r & 15; - - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); - _mm_storeu_ps(val_2, vs); - val_2 += _mm_popcnt_u32(r1); - - r2 = (r >> 4) & 15; - v = _mm_loadu_si128( - reinterpret_cast(&q_sparse_value[i2 + 4])); - vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); - _mm_storeu_ps(val_2, vs); - val_2 += _mm_popcnt_u32(r2); - } - - const uint16_t id1_max = m_sparse_index[i1 + 7]; - - if (id1_max <= q_sparse_index[i2 + 7]) { - i1 += 8; - if (i1 >= end1) goto do_scalar; - mm_index_m = _mm_loadu_si128( - reinterpret_cast(&m_sparse_index[i1])); - } - - if (id1_max >= q_sparse_index[i2 + 7]) { - i2 += 8; - if (i2 >= end2) goto do_scalar; - mm_index_q = _mm_loadu_si128( - reinterpret_cast(&q_sparse_index[i2])); - } - } - } - -do_scalar: - while (i1 < m_sparse_count && i2 < q_sparse_count) { - if (m_sparse_index[i1] == q_sparse_index[i2]) { - *val_1++ = m_sparse_value[i1]; - *val_2++ = q_sparse_value[i2]; - - ++i1; - ++i2; - } else if (m_sparse_index[i1] < q_sparse_index[i2]) { - ++i1; - } else { - ++i2; - } - } - - size_t res_num = val_1 - val_start_1; - - // if (res_num != val_2 - val_start_2) { - // std::cerr << "size mismatch!" << std::endl; - // } - - size_t res_num4 = res_num / 4 * 4; - - if (res_num4) { - __m128 sum128 = _mm_set1_ps(0); - - for (size_t k = 0; k < res_num4; k += 4) { - sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k), - _mm_loadu_ps(val_start_2 + k))); - } - - float __attribute__((aligned(16))) tmp_res[4]; - _mm_store_ps(tmp_res, sum128); - sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); - } - - for (size_t k = res_num4; k < res_num; ++k) - sum += val_start_1[k] * val_start_2[k]; - - return sum; -} -#else -float MipsInnerProductSparseInSegment(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const float *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const float *q_sparse_value) { - float sum = 0.0f; - - size_t m_i = 0; - size_t q_i = 0; - while (m_i < m_sparse_count && q_i < q_sparse_count) { - if (m_sparse_index[m_i] == q_sparse_index[q_i]) { - sum += m_sparse_value[m_i] * q_sparse_value[q_i]; - - ++m_i; - ++q_i; - } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { - ++m_i; - } else { - ++q_i; - } - } - - return sum; -} -#endif // __SSE4_1__ - -template <> -float MipsSquaredEuclideanSparseDistanceMatrix:: - ComputeInnerProductSparseInSegment(uint32_t m_sparse_count, - const uint16_t *m_sparse_index, - const ValueType *m_sparse_value, - uint32_t q_sparse_count, - const uint16_t *q_sparse_index, - const ValueType *q_sparse_value) { -#if defined(__SSE4_1__) - return MipsInnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#else - return MipsInnerProductSparseInSegment(m_sparse_count, m_sparse_index, - m_sparse_value, q_sparse_count, - q_sparse_index, q_sparse_value); -#endif -} - -#if defined(__ARM_NEON) -//! Compute the distance between matrix and query by SphericalInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { - float u2; - float v2; - float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); - - *out = ComputeSphericalInjection(sum, u2, v2, e2); -} - -//! Compute the distance between matrix and query by RepeatedQuadraticInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, - float *out) { - float u2; - float v2; - float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); - - sum = e2 * (u2 + v2 - 2 * sum); - u2 *= e2; - v2 *= e2; - for (size_t i = 0; i < m; ++i) { - sum += (u2 - v2) * (u2 - v2); - u2 = u2 * u2; - v2 = v2 * v2; - } - *out = sum; -} -#endif //__ARM_NEON - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx.cc new file mode 100644 index 0000000..cff60e8 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx.cc @@ -0,0 +1,114 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 4) << 4); + + __m256 ymm_sum_0 = _mm256_setzero_ps(); + __m256 ymm_sum_1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm1 = _mm256_setzero_ps(); + __m256 ymm_sum_norm2 = _mm256_setzero_ps(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_load_ps(lhs); + __m256 ymm_rhs_0 = _mm256_load_ps(rhs); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0); + __m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0); + __m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 8) { + __m256 ymm_lhs_0 = _mm256_loadu_ps(lhs); + __m256 ymm_rhs_0 = _mm256_loadu_ps(rhs); + ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + lhs += 8; + rhs += 8; + } + } + float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1)); + float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1); + float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2); + + switch (last - lhs) { + case 7: + FMA_FP32_GENERAL(lhs[6], rhs[6], result, norm1, norm2) + /* FALLTHRU */ + case 6: + FMA_FP32_GENERAL(lhs[5], rhs[5], result, norm1, norm2) + /* FALLTHRU */ + case 5: + FMA_FP32_GENERAL(lhs[4], rhs[4], result, norm1, norm2) + /* FALLTHRU */ + case 4: + FMA_FP32_GENERAL(lhs[3], rhs[3], result, norm1, norm2) + /* FALLTHRU */ + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __AVX__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.cc new file mode 100644 index 0000000..1ac56a2 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_avx512.cc @@ -0,0 +1,100 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX512F__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX512(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 5) << 5); + + __m512 zmm_sum_0 = _mm512_setzero_ps(); + __m512 zmm_sum_1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm1 = _mm512_setzero_ps(); + __m512 zmm_sum_norm2 = _mm512_setzero_ps(); + + if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_lhs_0 = _mm512_load_ps(lhs + 0); + __m512 zmm_lhs_1 = _mm512_load_ps(lhs + 16); + __m512 zmm_rhs_0 = _mm512_load_ps(rhs + 0); + __m512 zmm_rhs_1 = _mm512_load_ps(rhs + 16); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = _mm512_load_ps(lhs); + __m512 zmm_rhs_0 = _mm512_load_ps(rhs); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs + 0); + __m512 zmm_lhs_1 = _mm512_loadu_ps(lhs + 16); + __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs + 0); + __m512 zmm_rhs_1 = _mm512_loadu_ps(rhs + 16); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2) + } + + if (last >= last_aligned + 16) { + __m512 zmm_lhs_0 = _mm512_loadu_ps(lhs); + __m512 zmm_rhs_0 = _mm512_loadu_ps(rhs); + FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0) + FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1) + FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2) + lhs += 16; + rhs += 16; + } + } + + zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1); + if (lhs != last) { + __mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1); + __m512 zmm_undefined = _mm512_undefined_ps(); + __m512 zmm_lhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, lhs); + __m512 zmm_rhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, rhs); + FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0, mask); + FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1, mask); + FMA_MASK_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2, mask); + } + + *sql = HorizontalAdd_FP32_V512(zmm_sum_norm1); + *sqr = HorizontalAdd_FP32_V512(zmm_sum_norm2); + return HorizontalAdd_FP32_V512(zmm_sum_0); +} +#endif // __AVX512F__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc new file mode 100644 index 0000000..992da0d --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_dispatch.cc @@ -0,0 +1,136 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +float InnerProductAndSquaredNormNEON(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__AVX512F__) +float InnerProductAndSquaredNormAVX512(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__AVX__) +float InnerProductAndSquaredNormAVX(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE__) +float InnerProductAndSquaredNormSSE(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE4_1__) +float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value); +#endif + +float MipsInnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value); + +#if defined(__SSE__) +//! Compute the distance between matrix and query by SphericalInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F && dim > 15) { + sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); + } else +#endif // __AVX512F__ +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX && dim > 7) { + sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); + } else +#endif // __AVX__ + { + sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); + } + + *out = ComputeSphericalInjection(sum, u2, v2, e2); +} + +//! Compute the distance between matrix and query by RepeatedQuadraticInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, + float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F && dim > 15) { + sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2); + } else +#endif // __AVX512F__ +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX && dim > 7) { + sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); + } else +#endif // __AVX__ + { + sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); + } + + sum = e2 * (u2 + v2 - 2 * sum); + u2 *= e2; + v2 *= e2; + for (size_t i = 0; i < m; ++i) { + sum += (u2 - v2) * (u2 - v2); + u2 = u2 * u2; + v2 = v2 * v2; + } + *out = sum; +} +#endif // __SSE__ + +template <> +float MipsSquaredEuclideanSparseDistanceMatrix:: + ComputeInnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const ValueType *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const ValueType *q_sparse_value) { +#if defined(__SSE4_1__) + return MipsInnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#else + return MipsInnerProductSparseInSegment(m_sparse_count, m_sparse_index, + m_sparse_value, q_sparse_count, + q_sparse_index, q_sparse_value); +#endif +} + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc new file mode 100644 index 0000000..8e98922 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_neon.cc @@ -0,0 +1,105 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__ARM_NEON) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormNEON(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + float32x4_t v_sum_0 = vdupq_n_f32(0); + float32x4_t v_sum_1 = vdupq_n_f32(0); + float32x4_t v_sum_norm1 = vdupq_n_f32(0); + float32x4_t v_sum_norm2 = vdupq_n_f32(0); + + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + float32x4_t v_lhs_0 = vld1q_f32(lhs + 0); + float32x4_t v_lhs_1 = vld1q_f32(lhs + 4); + float32x4_t v_rhs_0 = vld1q_f32(rhs + 0); + float32x4_t v_rhs_1 = vld1q_f32(rhs + 4); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1); + } + if (last >= last_aligned + 4) { + float32x4_t v_lhs_0 = vld1q_f32(lhs); + float32x4_t v_rhs_0 = vld1q_f32(rhs); + v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0); + v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0); + v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0); + lhs += 4; + rhs += 4; + } + + float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1)); + float norm1 = vaddvq_f32(v_sum_norm1); + float norm2 = vaddvq_f32(v_sum_norm2); + switch (last - lhs) { + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} + +//! Compute the distance between matrix and query by SphericalInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { + float u2; + float v2; + float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); + + *out = ComputeSphericalInjection(sum, u2, v2, e2); +} + +//! Compute the distance between matrix and query by RepeatedQuadraticInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, + float *out) { + float u2; + float v2; + float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2); + + sum = e2 * (u2 + v2 - 2 * sum); + u2 *= e2; + v2 *= e2; + for (size_t i = 0; i < m; ++i) { + sum += (u2 - v2) * (u2 - v2); + u2 = u2 * u2; + v2 = v2 * v2; + } + *out = sum; +} +#endif //__ARM_NEON + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc new file mode 100644 index 0000000..43d8f9b --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_fp32_sse.cc @@ -0,0 +1,336 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_fp32.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormSSE(const float *lhs, const float *rhs, + size_t size, float *sql, float *sqr) { + const float *last = lhs + size; + const float *last_aligned = lhs + ((size >> 3) << 3); + + __m128 xmm_sum = _mm_setzero_ps(); + __m128 xmm_sum_norm1 = _mm_setzero_ps(); + __m128 xmm_sum_norm2 = _mm_setzero_ps(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_load_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_load_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_load_ps(rhs + 4); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 4) { + __m128 xmm_lhs_0 = _mm_load_ps(lhs); + __m128 xmm_rhs_0 = _mm_load_ps(rhs); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + lhs += 4; + rhs += 4; + } + } else { + for (; lhs != last_aligned; lhs += 8, rhs += 8) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0); + __m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0); + __m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 4) { + __m128 xmm_lhs_0 = _mm_loadu_ps(lhs); + __m128 xmm_rhs_0 = _mm_loadu_ps(rhs); + xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum); + xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + lhs += 4; + rhs += 4; + } + } + float result = HorizontalAdd_FP32_V128(xmm_sum); + float norm1 = HorizontalAdd_FP32_V128(xmm_sum_norm1); + float norm2 = HorizontalAdd_FP32_V128(xmm_sum_norm2); + + switch (last - lhs) { + case 3: + FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} + +#endif // __SSE__ + +// #if 1 +#if defined(__SSE4_1__) +const static __m128i SHUFFLE_MASK16[16] = { + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, -127, -127, -127, -127), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3, + 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, + -127, -127, 15, 14, 13, 12), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 3, 2, 1, 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 7, 6, 5, 4), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12, + 11, 10, 9, 8), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1, + 0), + _mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + 4), + _mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), +}; + +constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536; + +float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value) { + float sum = 0.0f; + + // size_t alloc_size = 0; + + size_t i1 = 0, i2 = 0; + size_t end1 = m_sparse_count / 8 * 8; + size_t end2 = q_sparse_count / 8 * 8; + + // std::vector mem1; + // std::vector mem2; + + float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH]; + float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH]; + + float *val_start_1 = fixed_buffer_1; + float *val_start_2 = fixed_buffer_2; + + // uint32_t max_count = std::max(m_sparse_count, q_sparse_count); + + // if (MAX_SPARSE_BUFFER_LENGTH < max_count) { + // mem1.reserve(max_count); + // mem2.reserve(max_count); + + // val_start_1 = mem1.data(); + // val_start_2 = mem2.data(); + // } + + float *val_1 = val_start_1; + float *val_2 = val_start_2; + + if (i1 < end1 && i2 < end2) { + while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + } + + while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + } + + __m128i mm_index_m = + _mm_loadu_si128(reinterpret_cast(&m_sparse_index[i1])); + __m128i mm_index_q = + _mm_loadu_si128(reinterpret_cast(&q_sparse_index[i2])); + + while (true) { +#ifdef DEBUG_PRINT + std::cout << "index 1: " << std::endl; + print_data16(&mm_index_m); + + std::cout << "index 2: " << std::endl; + print_data16(&mm_index_q); +#endif + + __m128i mm_cmp_res = + _mm_cmpistrm(mm_index_q, mm_index_m, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + +#ifdef DEBUG_PRINT + std::cout << "cmp res: " << std::endl; + print_data16(&mm_cmp_res); +#endif + + int r = _mm_extract_epi32(mm_cmp_res, 0); + + if (r) { + int r1 = r & 15; + + __m128i v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1])); + __m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); + + _mm_storeu_ps(val_1, vs); + val_1 += _mm_popcnt_u32(r1); + + int r2 = (r >> 4) & 15; + v = _mm_loadu_si128( + reinterpret_cast(&m_sparse_value[i1 + 4])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); + _mm_storeu_ps(val_1, vs); + val_1 += _mm_popcnt_u32(r2); + + mm_cmp_res = _mm_cmpistrm( + mm_index_m, mm_index_q, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + r = _mm_extract_epi32(mm_cmp_res, 0); + + r1 = r & 15; + + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1])); + _mm_storeu_ps(val_2, vs); + val_2 += _mm_popcnt_u32(r1); + + r2 = (r >> 4) & 15; + v = _mm_loadu_si128( + reinterpret_cast(&q_sparse_value[i2 + 4])); + vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2])); + _mm_storeu_ps(val_2, vs); + val_2 += _mm_popcnt_u32(r2); + } + + const uint16_t id1_max = m_sparse_index[i1 + 7]; + + if (id1_max <= q_sparse_index[i2 + 7]) { + i1 += 8; + if (i1 >= end1) goto do_scalar; + mm_index_m = _mm_loadu_si128( + reinterpret_cast(&m_sparse_index[i1])); + } + + if (id1_max >= q_sparse_index[i2 + 7]) { + i2 += 8; + if (i2 >= end2) goto do_scalar; + mm_index_q = _mm_loadu_si128( + reinterpret_cast(&q_sparse_index[i2])); + } + } + } + +do_scalar: + while (i1 < m_sparse_count && i2 < q_sparse_count) { + if (m_sparse_index[i1] == q_sparse_index[i2]) { + *val_1++ = m_sparse_value[i1]; + *val_2++ = q_sparse_value[i2]; + + ++i1; + ++i2; + } else if (m_sparse_index[i1] < q_sparse_index[i2]) { + ++i1; + } else { + ++i2; + } + } + + size_t res_num = val_1 - val_start_1; + + // if (res_num != val_2 - val_start_2) { + // std::cerr << "size mismatch!" << std::endl; + // } + + size_t res_num4 = res_num / 4 * 4; + + if (res_num4) { + __m128 sum128 = _mm_set1_ps(0); + + for (size_t k = 0; k < res_num4; k += 4) { + sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k), + _mm_loadu_ps(val_start_2 + k))); + } + + float __attribute__((aligned(16))) tmp_res[4]; + _mm_store_ps(tmp_res, sum128); + sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]); + } + + for (size_t k = res_num4; k < res_num; ++k) + sum += val_start_1[k] * val_start_2[k]; + + return sum; +} +#else +float MipsInnerProductSparseInSegment(uint32_t m_sparse_count, + const uint16_t *m_sparse_index, + const float *m_sparse_value, + uint32_t q_sparse_count, + const uint16_t *q_sparse_index, + const float *q_sparse_value) { + float sum = 0.0f; + + size_t m_i = 0; + size_t q_i = 0; + while (m_i < m_sparse_count && q_i < q_sparse_count) { + if (m_sparse_index[m_i] == q_sparse_index[q_i]) { + sum += m_sparse_value[m_i] * q_sparse_value[q_i]; + + ++m_i; + ++q_i; + } else if (m_sparse_index[m_i] < q_sparse_index[q_i]) { + ++m_i; + } else { + ++q_i; + } + } + + return sum; +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int4.cc b/src/ailego/math/mips_euclidean_distance_matrix_int4.cc deleted file mode 100644 index dc74b2d..0000000 --- a/src/ailego/math/mips_euclidean_distance_matrix_int4.cc +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_int8.i" -#include "inner_product_matrix.h" -#include "mips_euclidean_distance_matrix.h" -#include "norm_matrix.h" - -namespace zvec { -namespace ailego { - -#if defined(__SSE4_1__) -//! Four-bits Convert Table -static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = { - 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1, - 0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1}; -#endif // __SSE4_1__ - -#if defined(__SSE4_1__) -static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f); -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -static const __m128i INT4_LOOKUP_SSE = - _mm_load_si128((const __m128i *)Int4ConvertTable); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f); -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -static const __m256i INT4_LOOKUP_AVX = - _mm256_load_si256((const __m256i *)Int4ConvertTable); -#endif // __AVX2__ - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_INT4_GENERAL(lhs, rhs, sum, norm1, norm2) \ - { \ - sum += Int4MulTable[(((lhs) << 4) & 0xf0) | (((rhs) >> 0) & 0xf)] + \ - Int4MulTable[(((lhs) >> 0) & 0xf0) | (((rhs) >> 4) & 0xf)]; \ - norm1 += static_cast( \ - ((int8_t)((lhs) << 4) >> 4) * ((int8_t)((lhs) << 4) >> 4) + \ - ((int8_t)((lhs) & 0xf0) >> 4) * ((int8_t)((lhs) & 0xf0) >> 4)); \ - norm2 += static_cast( \ - ((int8_t)((rhs) << 4) >> 4) * ((int8_t)((rhs) << 4) >> 4) + \ - ((int8_t)((rhs) & 0xf0) >> 4) * ((int8_t)((rhs) & 0xf0) >> 4)); \ - } - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ - _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ - ONES_INT16_SSE), \ - xmm_sum) - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ - _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ - ONES_INT16_AVX), \ - ymm_sum) - -//! Compute the distance between matrix and query (SSE) -#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum_0, xmm_sum_norm1, \ - xmm_sum_norm2) \ - { \ - __m128i xmm_lhs_0 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \ - __m128i xmm_rhs_0 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \ - __m128i xmm_lhs_1 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \ - __m128i xmm_rhs_1 = _mm_shuffle_epi8( \ - INT4_LOOKUP_SSE, \ - _mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \ - FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); \ - FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); \ - FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); \ - FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum_0); \ - FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); \ - FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); \ - } - -//! Compute the distance between matrix and query (AVX) -#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, \ - ymm_sum_norm1, ymm_sum_norm2) \ - { \ - __m256i ymm_lhs_0 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \ - __m256i ymm_rhs_0 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \ - __m256i ymm_lhs_1 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \ - __m256i ymm_rhs_1 = _mm256_shuffle_epi8( \ - INT4_LOOKUP_AVX, \ - _mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \ - FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); \ - FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); \ - FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); \ - FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); \ - FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); \ - FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); \ - } - -#if defined(__SSE4_1__) -#if defined(__AVX2__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX(const uint8_t *lhs, - const uint8_t *rhs, - size_t size, float *sql, - float *sqr) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 5) << 5); - __m256i ymm_sum_0 = _mm256_setzero_si256(); - __m256i ymm_sum_1 = _mm256_setzero_si256(); - __m256i ymm_sum_norm1 = _mm256_setzero_si256(); - __m256i ymm_sum_norm2 = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); - FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1, - ymm_sum_norm2) - } - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - __m128i xmm_sum_norm1 = _mm_setzero_si128(); - __m128i xmm_sum_norm2 = _mm_setzero_si128(); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) - ymm_sum_0 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0); - ymm_sum_norm1 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1); - ymm_sum_norm2 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); - FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1, - ymm_sum_norm2) - } - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - __m128i xmm_sum = _mm_setzero_si128(); - __m128i xmm_sum_norm1 = _mm_setzero_si128(); - __m128i xmm_sum_norm2 = _mm_setzero_si128(); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) - ymm_sum_0 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0); - ymm_sum_norm1 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1); - ymm_sum_norm2 = _mm256_add_epi32( - _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2); - lhs += 16; - rhs += 16; - } - } - float result = static_cast( - HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); - float norm1 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm1)); - float norm2 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm2)); - - switch (last - lhs) { - case 15: - FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2) - /* FALLTHRU */ - case 14: - FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2) - /* FALLTHRU */ - case 13: - FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2) - /* FALLTHRU */ - case 12: - FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2) - /* FALLTHRU */ - case 11: - FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2) - /* FALLTHRU */ - case 10: - FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2) - /* FALLTHRU */ - case 9: - FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2) - /* FALLTHRU */ - case 8: - FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2) - /* FALLTHRU */ - case 7: - FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2) - /* FALLTHRU */ - case 6: - FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2) - /* FALLTHRU */ - case 5: - FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2) - /* FALLTHRU */ - case 4: - FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2) - /* FALLTHRU */ - case 3: - FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#else -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormSSE(const uint8_t *lhs, - const uint8_t *rhs, - size_t size, float *sql, - float *sqr) { - const uint8_t *last = lhs + size; - const uint8_t *last_aligned = lhs + ((size >> 4) << 4); - __m128i xmm_sum = _mm_setzero_si128(); - __m128i xmm_sum_norm1 = _mm_setzero_si128(); - __m128i xmm_sum_norm2 = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) - } - } else { - for (; lhs != last_aligned; lhs += 16, rhs += 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); - FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) - } - } - float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); - float norm1 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm1)); - float norm2 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm2)); - - switch (last - lhs) { - case 15: - FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2) - /* FALLTHRU */ - case 14: - FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2) - /* FALLTHRU */ - case 13: - FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2) - /* FALLTHRU */ - case 12: - FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2) - /* FALLTHRU */ - case 11: - FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2) - /* FALLTHRU */ - case 10: - FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2) - /* FALLTHRU */ - case 9: - FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2) - /* FALLTHRU */ - case 8: - FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2) - /* FALLTHRU */ - case 7: - FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2) - /* FALLTHRU */ - case 6: - FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2) - /* FALLTHRU */ - case 5: - FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2) - /* FALLTHRU */ - case 4: - FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2) - /* FALLTHRU */ - case 3: - FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __AVX2__ - -//! Compute the distance between matrix and query by SphericalInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX2__) - sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2); -#else - sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2); -#endif - - *out = ComputeSphericalInjection(sum, u2, v2, e2); -} - -//! Compute the distance between matrix and query by RepeatedQuadraticInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, - float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX2__) - sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2); -#else - sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2); -#endif - - sum = e2 * (u2 + v2 - 2 * sum); - u2 *= e2; - v2 *= e2; - for (size_t i = 0; i < m; ++i) { - sum += (u2 - v2) * (u2 - v2); - u2 = u2 * u2; - v2 = v2 * v2; - } - *out = sum; -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int4_avx2.cc b/src/ailego/math/mips_euclidean_distance_matrix_int4_avx2.cc new file mode 100644 index 0000000..95a3f00 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int4_avx2.cc @@ -0,0 +1,140 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_mips_utility.i" +#include "inner_product_matrix.h" +#include "mips_euclidean_distance_matrix.h" +#include "norm_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX(const uint8_t *lhs, const uint8_t *rhs, + size_t size, float *sql, float *sqr) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 5) << 5); + __m256i ymm_sum_0 = _mm256_setzero_si256(); + __m256i ymm_sum_1 = _mm256_setzero_si256(); + __m256i ymm_sum_norm1 = _mm256_setzero_si256(); + __m256i ymm_sum_norm2 = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs)); + FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1, + ymm_sum_norm2) + } + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + __m128i xmm_sum_norm1 = _mm_setzero_si128(); + __m128i xmm_sum_norm2 = _mm_setzero_si128(); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0); + ymm_sum_norm1 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1); + ymm_sum_norm2 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs)); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs)); + FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1, + ymm_sum_norm2) + } + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + __m128i xmm_sum = _mm_setzero_si128(); + __m128i xmm_sum_norm1 = _mm_setzero_si128(); + __m128i xmm_sum_norm2 = _mm_setzero_si128(); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) + ymm_sum_0 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0); + ymm_sum_norm1 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1); + ymm_sum_norm2 = _mm256_add_epi32( + _mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2); + lhs += 16; + rhs += 16; + } + } + float result = static_cast( + HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); + float norm1 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm1)); + float norm2 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm2)); + + switch (last - lhs) { + case 15: + FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2) + /* FALLTHRU */ + case 14: + FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2) + /* FALLTHRU */ + case 13: + FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2) + /* FALLTHRU */ + case 12: + FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2) + /* FALLTHRU */ + case 11: + FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2) + /* FALLTHRU */ + case 10: + FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2) + /* FALLTHRU */ + case 9: + FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2) + /* FALLTHRU */ + case 8: + FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2) + /* FALLTHRU */ + case 7: + FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2) + /* FALLTHRU */ + case 6: + FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2) + /* FALLTHRU */ + case 5: + FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2) + /* FALLTHRU */ + case 4: + FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2) + /* FALLTHRU */ + case 3: + FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __AVX2__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int4_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_int4_dispatch.cc new file mode 100644 index 0000000..c967f83 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int4_dispatch.cc @@ -0,0 +1,83 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "inner_product_matrix.h" +#include "mips_euclidean_distance_matrix.h" +#include "norm_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX__) +float InnerProductAndSquaredNormAVX(const uint8_t *lhs, const uint8_t *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE__) +float InnerProductAndSquaredNormSSE(const uint8_t *lhs, const uint8_t *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query by SphericalInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2); + } else +#endif + { + sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2); + } + + *out = ComputeSphericalInjection(sum, u2, v2, e2); +} + +//! Compute the distance between matrix and query by RepeatedQuadraticInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, + float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2); + } else +#endif + { + sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2); + } + + sum = e2 * (u2 + v2 - 2 * sum); + u2 *= e2; + v2 *= e2; + for (size_t i = 0; i < m; ++i) { + sum += (u2 - v2) * (u2 - v2); + u2 = u2 * u2; + v2 = v2 * v2; + } + *out = sum; +} +#endif + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int4_sse.cc b/src/ailego/math/mips_euclidean_distance_matrix_int4_sse.cc new file mode 100644 index 0000000..139b14c --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int4_sse.cc @@ -0,0 +1,104 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_mips_utility.i" +#include "inner_product_matrix.h" +#include "mips_euclidean_distance_matrix.h" +#include "norm_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormSSE(const uint8_t *lhs, const uint8_t *rhs, + size_t size, float *sql, float *sqr) { + const uint8_t *last = lhs + size; + const uint8_t *last_aligned = lhs + ((size >> 4) << 4); + __m128i xmm_sum = _mm_setzero_si128(); + __m128i xmm_sum_norm1 = _mm_setzero_si128(); + __m128i xmm_sum_norm2 = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs)); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) + } + } else { + for (; lhs != last_aligned; lhs += 16, rhs += 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs)); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs)); + FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2) + } + } + float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); + float norm1 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm1)); + float norm2 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm2)); + + switch (last - lhs) { + case 15: + FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2) + /* FALLTHRU */ + case 14: + FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2) + /* FALLTHRU */ + case 13: + FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2) + /* FALLTHRU */ + case 12: + FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2) + /* FALLTHRU */ + case 11: + FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2) + /* FALLTHRU */ + case 10: + FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2) + /* FALLTHRU */ + case 9: + FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2) + /* FALLTHRU */ + case 8: + FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2) + /* FALLTHRU */ + case 7: + FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2) + /* FALLTHRU */ + case 6: + FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2) + /* FALLTHRU */ + case 5: + FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2) + /* FALLTHRU */ + case 4: + FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2) + /* FALLTHRU */ + case 3: + FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int8.cc b/src/ailego/math/mips_euclidean_distance_matrix_int8.cc deleted file mode 100644 index accf3d1..0000000 --- a/src/ailego/math/mips_euclidean_distance_matrix_int8.cc +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2025-present the zvec project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "distance_matrix_accum_int8.i" -#include "mips_euclidean_distance_matrix.h" - -namespace zvec { -namespace ailego { - -#if defined(__SSE4_1__) -static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001); -#endif // __SSE4_1__ - -#if defined(__AVX2__) -static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001); -#endif // __AVX2__ - -//! Calculate Fused-Multiply-Add (GENERAL) -#define FMA_INT8_GENERAL(lhs, rhs, sum, norm1, norm2) \ - { \ - sum += static_cast(lhs * rhs); \ - norm1 += static_cast(lhs * lhs); \ - norm2 += static_cast(rhs * rhs); \ - } - -//! Calculate Fused-Multiply-Add (AVX) -#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \ - ymm_sum = _mm256_add_epi32( \ - _mm256_madd_epi16( \ - _mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \ - _mm256_sign_epi8(ymm_lhs, ymm_rhs)), \ - ONES_INT16_AVX), \ - ymm_sum) -#define FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum) \ - ymm_sum = _mm256_add_epi32( \ - _mm256_set_m128i( \ - _mm_setzero_si128(), \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ - _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ - ONES_INT16_SSE)), \ - ymm_sum) - -//! Calculate Fused-Multiply-Add (SSE) -#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \ - xmm_sum = _mm_add_epi32( \ - _mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \ - _mm_sign_epi8(xmm_lhs, xmm_rhs)), \ - ONES_INT16_SSE), \ - xmm_sum) - -#if defined(__SSE4_1__) -#if defined(__AVX2__) -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormAVX(const int8_t *lhs, - const int8_t *rhs, - size_t size, float *sql, - float *sqr) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 6) << 6); - - __m256i ymm_sum_0 = _mm256_setzero_si256(); - __m256i ymm_sum_1 = _mm256_setzero_si256(); - __m256i ymm_sum_norm1 = _mm256_setzero_si256(); - __m256i ymm_sum_norm2 = _mm256_setzero_si256(); - - if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); - FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); - FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0); - FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1); - FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2); - lhs += 32; - rhs += 32; - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0); - FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1); - FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 64, rhs += 64) { - __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); - __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); - __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); - __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); - FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); - FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); - FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); - FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); - FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); - FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); - } - - if (last >= last_aligned + 32) { - __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); - __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); - FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0); - FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1); - FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2); - lhs += 32; - rhs += 32; - } - - if (last >= lhs + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0); - FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1); - FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2); - lhs += 16; - rhs += 16; - } - } - float result = static_cast( - HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); - float norm1 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm1)); - float norm2 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm2)); - - switch (last - lhs) { - case 15: - FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2) - /* FALLTHRU */ - case 14: - FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2) - /* FALLTHRU */ - case 13: - FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2) - /* FALLTHRU */ - case 12: - FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2) - /* FALLTHRU */ - case 11: - FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2) - /* FALLTHRU */ - case 10: - FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2) - /* FALLTHRU */ - case 9: - FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2) - /* FALLTHRU */ - case 8: - FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2) - /* FALLTHRU */ - case 7: - FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2) - /* FALLTHRU */ - case 6: - FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2) - /* FALLTHRU */ - case 5: - FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2) - /* FALLTHRU */ - case 4: - FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2) - /* FALLTHRU */ - case 3: - FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#else -//! Compute the Inner Product between p and q, and each Squared L2-Norm value -static inline float InnerProductAndSquaredNormSSE(const int8_t *lhs, - const int8_t *rhs, - size_t size, float *sql, - float *sqr) { - const int8_t *last = lhs + size; - const int8_t *last_aligned = lhs + ((size >> 5) << 5); - - __m128i xmm_sum = _mm_setzero_si128(); - __m128i xmm_sum_norm1 = _mm_setzero_si128(); - __m128i xmm_sum_norm2 = _mm_setzero_si128(); - - if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); - FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum); - FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum); - FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); - FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum); - FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2); - lhs += 16; - rhs += 16; - } - } else { - for (; lhs != last_aligned; lhs += 32, rhs += 32) { - __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); - __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); - __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); - __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); - FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum); - FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); - FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum); - FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); - } - - if (last >= last_aligned + 16) { - __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); - __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); - FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum); - FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1); - FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2); - lhs += 16; - rhs += 16; - } - } - float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); - float norm1 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm1)); - float norm2 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm2)); - - switch (last - lhs) { - case 15: - FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2) - /* FALLTHRU */ - case 14: - FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2) - /* FALLTHRU */ - case 13: - FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2) - /* FALLTHRU */ - case 12: - FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2) - /* FALLTHRU */ - case 11: - FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2) - /* FALLTHRU */ - case 10: - FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2) - /* FALLTHRU */ - case 9: - FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2) - /* FALLTHRU */ - case 8: - FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2) - /* FALLTHRU */ - case 7: - FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2) - /* FALLTHRU */ - case 6: - FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2) - /* FALLTHRU */ - case 5: - FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2) - /* FALLTHRU */ - case 4: - FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2) - /* FALLTHRU */ - case 3: - FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2) - /* FALLTHRU */ - case 2: - FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2) - /* FALLTHRU */ - case 1: - FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2) - } - *sql = norm1; - *sqr = norm2; - return result; -} -#endif // __AVX2__ - -//! Compute the distance between matrix and query by SphericalInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX2__) - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); -#else - sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); -#endif - - *out = ComputeSphericalInjection(sum, u2, v2, e2); -} - -//! Compute the distance between matrix and query by RepeatedQuadraticInjection -void MipsSquaredEuclideanDistanceMatrix::Compute( - const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, - float *out) { - float u2; - float v2; - float sum; - -#if defined(__AVX2__) - sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2); -#else - sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); -#endif - - sum = e2 * (u2 + v2 - 2 * sum); - u2 *= e2; - v2 *= e2; - for (size_t i = 0; i < m; ++i) { - sum += (u2 - v2) * (u2 - v2); - u2 = u2 * u2; - v2 = v2 * v2; - } - *out = sum; -} -#endif // __SSE4_1__ - -} // namespace ailego -} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int8_avx2.cc b/src/ailego/math/mips_euclidean_distance_matrix_int8_avx2.cc new file mode 100644 index 0000000..0b96953 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int8_avx2.cc @@ -0,0 +1,159 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormAVX2(const int8_t *lhs, const int8_t *rhs, + size_t size, float *sql, float *sqr) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 6) << 6); + + __m256i ymm_sum_0 = _mm256_setzero_si256(); + __m256i ymm_sum_1 = _mm256_setzero_si256(); + __m256i ymm_sum_norm1 = _mm256_setzero_si256(); + __m256i ymm_sum_norm2 = _mm256_setzero_si256(); + + if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32)); + FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs); + FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0); + FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1); + FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0); + FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1); + FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 64, rhs += 64) { + __m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0)); + __m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32)); + __m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0)); + __m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32)); + FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); + FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); + FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); + FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); + FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); + FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); + } + + if (last >= last_aligned + 32) { + __m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs); + __m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs); + FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0); + FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1); + FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2); + lhs += 32; + rhs += 32; + } + + if (last >= lhs + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0); + FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1); + FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2); + lhs += 16; + rhs += 16; + } + } + float result = static_cast( + HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1))); + float norm1 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm1)); + float norm2 = static_cast(HorizontalAdd_INT32_V256(ymm_sum_norm2)); + + switch (last - lhs) { + case 15: + FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2) + /* FALLTHRU */ + case 14: + FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2) + /* FALLTHRU */ + case 13: + FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2) + /* FALLTHRU */ + case 12: + FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2) + /* FALLTHRU */ + case 11: + FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2) + /* FALLTHRU */ + case 10: + FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2) + /* FALLTHRU */ + case 9: + FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2) + /* FALLTHRU */ + case 8: + FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2) + /* FALLTHRU */ + case 7: + FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2) + /* FALLTHRU */ + case 6: + FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2) + /* FALLTHRU */ + case 5: + FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2) + /* FALLTHRU */ + case 4: + FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2) + /* FALLTHRU */ + case 3: + FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __AVX2__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int8_dispatch.cc b/src/ailego/math/mips_euclidean_distance_matrix_int8_dispatch.cc new file mode 100644 index 0000000..35b4a8c --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int8_dispatch.cc @@ -0,0 +1,81 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__AVX2__) +float InnerProductAndSquaredNormAVX2(const int8_t *lhs, const int8_t *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE__) +float InnerProductAndSquaredNormSSE(const int8_t *lhs, const int8_t *rhs, + size_t size, float *sql, float *sqr); +#endif + +#if defined(__SSE4_1__) +//! Compute the distance between matrix and query by SphericalInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + sum = InnerProductAndSquaredNormAVX2(p, q, dim, &u2, &v2); + } else +#endif + { + sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); + } + + *out = ComputeSphericalInjection(sum, u2, v2, e2); +} + +//! Compute the distance between matrix and query by RepeatedQuadraticInjection +void MipsSquaredEuclideanDistanceMatrix::Compute( + const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2, + float *out) { + float u2{0.0f}; + float v2{0.0f}; + float sum{0.0f}; + +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + sum = InnerProductAndSquaredNormAVX2(p, q, dim, &u2, &v2); + } else +#endif + { + sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2); + } + + sum = e2 * (u2 + v2 - 2 * sum); + u2 *= e2; + v2 *= e2; + for (size_t i = 0; i < m; ++i) { + sum += (u2 - v2) * (u2 - v2); + u2 = u2 * u2; + v2 = v2 * v2; + } + *out = sum; +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/mips_euclidean_distance_matrix_int8_sse.cc b/src/ailego/math/mips_euclidean_distance_matrix_int8_sse.cc new file mode 100644 index 0000000..a0d6192 --- /dev/null +++ b/src/ailego/math/mips_euclidean_distance_matrix_int8_sse.cc @@ -0,0 +1,137 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "distance_matrix_accum_int8.i" +#include "distance_matrix_mips_utility.i" +#include "mips_euclidean_distance_matrix.h" + +namespace zvec { +namespace ailego { + +#if defined(__SSE4_1__) +//! Compute the Inner Product between p and q, and each Squared L2-Norm value +float InnerProductAndSquaredNormSSE(const int8_t *lhs, const int8_t *rhs, + size_t size, float *sql, float *sqr) { + const int8_t *last = lhs + size; + const int8_t *last_aligned = lhs + ((size >> 5) << 5); + + __m128i xmm_sum = _mm_setzero_si128(); + __m128i xmm_sum_norm1 = _mm_setzero_si128(); + __m128i xmm_sum_norm2 = _mm_setzero_si128(); + + if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16)); + FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum); + FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum); + FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs); + FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum); + FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2); + lhs += 16; + rhs += 16; + } + } else { + for (; lhs != last_aligned; lhs += 32, rhs += 32) { + __m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0)); + __m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16)); + __m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0)); + __m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16)); + FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum); + FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); + FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum); + FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); + } + + if (last >= last_aligned + 16) { + __m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs); + __m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs); + FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum); + FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1); + FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2); + lhs += 16; + rhs += 16; + } + } + float result = static_cast(HorizontalAdd_INT32_V128(xmm_sum)); + float norm1 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm1)); + float norm2 = static_cast(HorizontalAdd_INT32_V128(xmm_sum_norm2)); + + switch (last - lhs) { + case 15: + FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2) + /* FALLTHRU */ + case 14: + FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2) + /* FALLTHRU */ + case 13: + FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2) + /* FALLTHRU */ + case 12: + FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2) + /* FALLTHRU */ + case 11: + FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2) + /* FALLTHRU */ + case 10: + FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2) + /* FALLTHRU */ + case 9: + FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2) + /* FALLTHRU */ + case 8: + FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2) + /* FALLTHRU */ + case 7: + FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2) + /* FALLTHRU */ + case 6: + FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2) + /* FALLTHRU */ + case 5: + FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2) + /* FALLTHRU */ + case 4: + FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2) + /* FALLTHRU */ + case 3: + FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2) + /* FALLTHRU */ + case 2: + FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2) + /* FALLTHRU */ + case 1: + FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2) + } + *sql = norm1; + *sqr = norm2; + return result; +} +#endif // __SSE4_1__ + +} // namespace ailego +} // namespace zvec \ No newline at end of file diff --git a/src/ailego/math/norm1_matrix_fp16.cc b/src/ailego/math/norm1_matrix_fp16.cc index ce36de7..9bb8620 100644 --- a/src/ailego/math/norm1_matrix_fp16.cc +++ b/src/ailego/math/norm1_matrix_fp16.cc @@ -73,13 +73,17 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP16_1_NEON(m, dim, out, ) -#elif defined(__AVX512F__) - NORM_FP16_1_AVX512(m, dim, out, ) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP16_1_AVX512(m, dim, out, ) + return; + } +#endif NORM_FP16_1_AVX(m, dim, out, ) #endif } #endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/ailego/math/norm1_matrix_fp32.cc b/src/ailego/math/norm1_matrix_fp32.cc index 15bd3f7..c1ec366 100644 --- a/src/ailego/math/norm1_matrix_fp32.cc +++ b/src/ailego/math/norm1_matrix_fp32.cc @@ -26,18 +26,15 @@ namespace ailego { #define NORM_FP32_STEP_NEON SA_FP32_NEON #if defined(__SSE__) -static const __m128 ABS_MASK_FP32_SSE = - _mm_castsi128_ps(_mm_set1_epi32(0x7fffffffu)); +#define ABS_MASK_FP32_SSE _mm_castsi128_ps(_mm_set1_epi32(0x7fffffffu)) #endif // __SSE__ #if defined(__AVX__) -static const __m256 ABS_MASK_FP32_AVX = - _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffffu)); +#define ABS_MASK_FP32_AVX _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffffu)) #endif // __AVX__ #if defined(__AVX512F__) -static const __m512 ABS_MASK_FP32_AVX512 = - _mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffffu)); +#define ABS_MASK_FP32_AVX512 _mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffffu)) #endif // __AVX512F__ //! Calculate sum of absolute (GENERAL) @@ -64,15 +61,23 @@ void Norm1Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP32_1_NEON(m, dim, out, ) -#elif defined(__AVX512F__) - NORM_FP32_1_AVX512(m, dim, out, ) -#elif defined(__AVX__) - NORM_FP32_1_AVX(m, dim, out, ) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP32_1_AVX512(m, dim, out, ) + return; + } +#endif +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + NORM_FP32_1_AVX(m, dim, out, ) + return; + } +#endif NORM_FP32_1_SSE(m, dim, out, ) #endif } #endif // __SSE__ || (__ARM_NEON && __aarch64__) } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/ailego/math/norm2_matrix_fp16.cc b/src/ailego/math/norm2_matrix_fp16.cc index 28dbadb..37c3313 100644 --- a/src/ailego/math/norm2_matrix_fp16.cc +++ b/src/ailego/math/norm2_matrix_fp16.cc @@ -58,9 +58,13 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP16_1_NEON(m, dim, out, std::sqrt) -#elif defined(__AVX512F__) - NORM_FP16_1_AVX512(m, dim, out, std::sqrt) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP16_1_AVX512(m, dim, out, std::sqrt) + return; + } +#endif NORM_FP16_1_AVX(m, dim, out, std::sqrt) #endif } @@ -70,9 +74,13 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP16_1_NEON(m, dim, out, ) -#elif defined(__AVX512F__) - NORM_FP16_1_AVX512(m, dim, out, ) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP16_1_AVX512(m, dim, out, ) + return; + } +#endif NORM_FP16_1_AVX(m, dim, out, ) #endif } diff --git a/src/ailego/math/norm2_matrix_fp32.cc b/src/ailego/math/norm2_matrix_fp32.cc index b5dcf2f..8cc76c1 100644 --- a/src/ailego/math/norm2_matrix_fp32.cc +++ b/src/ailego/math/norm2_matrix_fp32.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "norm2_matrix.h" #include "norm_matrix_fp32.i" @@ -48,11 +49,19 @@ void Norm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP32_1_NEON(m, dim, out, std::sqrt) -#elif defined(__AVX512F__) - NORM_FP32_1_AVX512(m, dim, out, std::sqrt) -#elif defined(__AVX__) - NORM_FP32_1_AVX(m, dim, out, std::sqrt) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP32_1_AVX512(m, dim, out, std::sqrt) + return; + } +#endif +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + NORM_FP32_1_AVX(m, dim, out, std::sqrt) + return; + } +#endif NORM_FP32_1_SSE(m, dim, out, std::sqrt) #endif } @@ -62,15 +71,23 @@ void SquaredNorm2Matrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) NORM_FP32_1_NEON(m, dim, out, ) -#elif defined(__AVX512F__) - NORM_FP32_1_AVX512(m, dim, out, ) -#elif defined(__AVX__) - NORM_FP32_1_AVX(m, dim, out, ) #else +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + NORM_FP32_1_AVX512(m, dim, out, ) + return; + } +#endif +#if defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + NORM_FP32_1_AVX(m, dim, out, ) + return; + } +#endif NORM_FP32_1_SSE(m, dim, out, ) #endif } #endif // __SSE__ || (__ARM_NEON && __aarch64__) } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/ailego/math_batch/distance_batch.h b/src/ailego/math_batch/distance_batch.h index c762a25..17a2190 100644 --- a/src/ailego/math_batch/distance_batch.h +++ b/src/ailego/math_batch/distance_batch.h @@ -19,10 +19,8 @@ #include "cosine_distance_batch.h" #include "inner_product_distance_batch.h" - namespace zvec::ailego { - template < template class DistanceType, typename ValueType, size_t BatchSize, size_t PrefetchStep, typename = void> diff --git a/src/ailego/math_batch/inner_product_distance_batch.h b/src/ailego/math_batch/inner_product_distance_batch.h index f579949..f42345b 100644 --- a/src/ailego/math_batch/inner_product_distance_batch.h +++ b/src/ailego/math_batch/inner_product_distance_batch.h @@ -15,20 +15,28 @@ #pragma once #include -#include #include #include #include #include -#include "inner_product_distance_batch_impl.h" -#include "inner_product_distance_batch_impl_fp16.h" -#include "inner_product_distance_batch_impl_int8.h" namespace zvec::ailego::DistanceBatch { template struct InnerProductDistanceBatch; +template +static void compute_one_to_many_inner_product_fallback( + const ValueType *query, const ValueType **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + for (size_t j = 0; j < BatchSize; ++j) { + sums[j] = 0.0; + InnerProductMatrix::Compute(ptrs[j], query, dim, sums + j); + ailego_prefetch(&prefetch_ptrs[j]); + } +} + // Function template partial specialization is not allowed, // therefore the wrapper struct is required. template @@ -38,98 +46,14 @@ struct InnerProductDistanceBatchImpl { const ValueType *query, const ValueType **ptrs, std::array &prefetch_ptrs, size_t dim, float *sums) { - return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); + return compute_one_to_many_inner_product_fallback(query, ptrs, + prefetch_ptrs, dim, sums); } static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() { return nullptr; } }; -template -struct InnerProductDistanceBatchImpl { - using ValueType = float; - static void compute_one_to_many( - const ValueType *query, const ValueType **ptrs, - std::array &prefetch_ptrs, size_t dim, - float *sums) { -#if defined(__AVX2__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { - return compute_one_to_many_avx2_fp32( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif - return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); - } - - static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() { - return nullptr; - } -}; - -template -struct InnerProductDistanceBatchImpl { - using ValueType = int8_t; - static void compute_one_to_many( - const int8_t *query, const int8_t **ptrs, - std::array &prefetch_ptrs, size_t dim, - float *sums) { -// #if defined(__AVX512BW__) // TODO: this version is problematic -// return compute_one_to_many_avx512_int8( -// query, ptrs, prefetch_ptrs, dim, sums); -#if defined(__AVX512VNNI__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { - return compute_one_to_many_avx512_vnni_int8( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif -#if defined(__AVX2__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { - return compute_one_to_many_avx2_int8( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif - return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); - } - - static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() { -#if defined(__AVX512VNNI__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { - return compute_one_to_many_avx512_vnni_int8_query_preprocess; - } -#endif - return nullptr; - } -}; - -template -struct InnerProductDistanceBatchImpl { - using ValueType = ailego::Float16; - static void compute_one_to_many( - const ailego::Float16 *query, const ailego::Float16 **ptrs, - std::array &prefetch_ptrs, size_t dim, - float *sums) { -#if defined(__AVX512FP16__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { - return compute_one_to_many_avx512fp16_fp16( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif -#if defined(__AVX512F__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { - return compute_one_to_many_avx512f_fp16( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif -#if defined(__AVX2__) - if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { - return compute_one_to_many_avx2_fp16( - query, ptrs, prefetch_ptrs, dim, sums); - } -#endif - return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); - } -}; - template struct InnerProductDistanceBatch { using ValueType = typename std::remove_cv::type; @@ -163,4 +87,56 @@ struct InnerProductDistanceBatch { } }; +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = ailego::Float16; + static void compute_one_to_many( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums); +}; + +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = float; + static void compute_one_to_many(const float *query, const float **ptrs, + std::array &prefetch_ptrs, + size_t dim, float *sums); +}; + +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = int8_t; + static void compute_one_to_many(const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, + size_t dim, float *sums); + + static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc(); +}; + +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = ailego::Float16; + static void compute_one_to_many( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums); +}; + +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = float; + static void compute_one_to_many(const float *query, const float **ptrs, + std::array &prefetch_ptrs, + size_t dim, float *sums); +}; + +template <> +struct InnerProductDistanceBatchImpl { + using ValueType = int8_t; + static void compute_one_to_many(const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, + size_t dim, float *sums); +}; + } // namespace zvec::ailego::DistanceBatch diff --git a/src/ailego/math_batch/inner_product_distance_batch_dispatch.cc b/src/ailego/math_batch/inner_product_distance_batch_dispatch.cc new file mode 100644 index 0000000..7837662 --- /dev/null +++ b/src/ailego/math_batch/inner_product_distance_batch_dispatch.cc @@ -0,0 +1,228 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include "inner_product_distance_batch.h" + +namespace zvec::ailego::DistanceBatch { + +#if defined(__AVX512VNNI__) +void compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess( + void *query, size_t dim); + +void compute_one_to_many_inner_product_avx512_vnni_int8_1( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); + +void compute_one_to_many_inner_product_avx512_vnni_int8_12( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); +#endif + +#if defined(__AVX512FP16__) +void compute_one_to_many_inner_product_avx512fp16_fp16_1( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); + +void compute_one_to_many_inner_product_avx512fp16_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); +#endif //__AVX512FP16__ + +#if defined(__AVX512F__) +void compute_one_to_many_inner_product_avx512f_fp16_1( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); + +void compute_one_to_many_inner_product_avx512f_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); +#endif //__AVX512F__ + +#if defined(__AVX2__) +void compute_one_to_many_inner_product_avx2_fp32_1( + const float *query, const float **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); + +void compute_one_to_many_inner_product_avx2_fp16_1( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); + +void compute_one_to_many_inner_product_avx2_int8_1( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); + +void compute_one_to_many_inner_product_avx2_fp32_12( + const float *query, const float **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); + +void compute_one_to_many_inner_product_avx2_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results); + +void compute_one_to_many_inner_product_avx2_int8_12( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results); +#endif + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const ValueType *query, const ValueType **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_fp32_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + return compute_one_to_many_inner_product_avx512fp16_fp16_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return compute_one_to_many_inner_product_avx512f_fp16_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_fp16_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { +// #if defined(__AVX512BW__) // TODO: this version is problematic +// return compute_one_to_many_avx512_int8( +// query, ptrs, prefetch_ptrs, dim, sums); +#if defined(__AVX512VNNI__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + return compute_one_to_many_inner_product_avx512_vnni_int8_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_int8_1( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +DistanceBatchQueryPreprocessFunc +InnerProductDistanceBatchImpl::GetQueryPreprocessFunc() { +#if defined(__AVX512VNNI__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + return compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess; + } +#endif + return nullptr; +} + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const ValueType *query, const ValueType **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_fp32_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + return compute_one_to_many_inner_product_avx512fp16_fp16_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return compute_one_to_many_inner_product_avx512f_fp16_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_fp16_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +void InnerProductDistanceBatchImpl::compute_one_to_many( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { +// #if defined(__AVX512BW__) // TODO: this version is problematic +// return compute_one_to_many_avx512_int8( +// query, ptrs, prefetch_ptrs, dim, sums); +#if defined(__AVX512VNNI__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + return compute_one_to_many_inner_product_avx512_vnni_int8_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_inner_product_avx2_int8_12( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs, + dim, sums); +} + +} // namespace zvec::ailego::DistanceBatch diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx2.cc b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx2.cc new file mode 100644 index 0000000..59320de --- /dev/null +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx2.cc @@ -0,0 +1,110 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +namespace zvec::ailego::DistanceBatch { + +#if defined(__AVX2__) + +template +static std::enable_if_t, void> +compute_one_to_many_inner_product_avx2_fp16( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, + size_t dimensionality, float *results) { + std::array<__m256, dp_batch> accs; + + for (size_t i = 0; i < dp_batch; ++i) { + accs[i] = _mm256_setzero_ps(); + } + + size_t dim = 0; + for (; dim + 16 <= dimensionality; dim += 16) { + __m256i q = + _mm256_loadu_si256(reinterpret_cast(query + dim)); + + __m256 q1 = _mm256_cvtph_ps(_mm256_castsi256_si128(q)); + __m256 q2 = _mm256_cvtph_ps(_mm256_extractf128_si256(q, 1)); + + std::array<__m256, dp_batch> data_regs_1; + std::array<__m256, dp_batch> data_regs_2; + for (size_t i = 0; i < dp_batch; ++i) { + __m256i m = + _mm256_loadu_si256(reinterpret_cast(ptrs[i] + dim)); + + data_regs_1[i] = _mm256_cvtph_ps(_mm256_castsi256_si128(m)); + data_regs_2[i] = _mm256_cvtph_ps(_mm256_extractf128_si256(m, 1)); + } + + if (prefetch_ptrs[0]) { + for (size_t i = 0; i < dp_batch; ++i) { + ailego_prefetch(prefetch_ptrs[i] + dim); + } + } + + for (size_t i = 0; i < dp_batch; ++i) { + accs[i] = _mm256_fmadd_ps(q1, data_regs_1[i], accs[i]); + accs[i] = _mm256_fmadd_ps(q2, data_regs_2[i], accs[i]); + } + } + + if (dim + 8 <= dimensionality) { + __m256 q = _mm256_cvtph_ps( + _mm_loadu_si128(reinterpret_cast(query + dim))); + + std::array<__m256, dp_batch> data_regs; + for (size_t i = 0; i < dp_batch; ++i) { + data_regs[i] = _mm256_cvtph_ps( + _mm_loadu_si128(reinterpret_cast(ptrs[i] + dim))); + accs[i] = _mm256_fmadd_ps(q, data_regs[i], accs[i]); + } + + dim += 8; + } + + for (size_t i = 0; i < dp_batch; ++i) { + results[i] = HorizontalAdd_FP32_V256(accs[i]); + } + + for (; dim < dimensionality; ++dim) { + for (size_t i = 0; i < dp_batch; ++i) { + results[i] += (*(query + dim)) * (*(ptrs[i] + dim)); + } + } +} + +void compute_one_to_many_inner_product_avx2_fp16_1( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx2_fp16( + query, ptrs, prefetch_ptrs, dim, sums); +} + +void compute_one_to_many_inner_product_avx2_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx2_fp16( + query, ptrs, prefetch_ptrs, dim, sums); +} + +#endif + +} // namespace zvec::ailego::DistanceBatch \ No newline at end of file diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx512.cc similarity index 70% rename from src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h rename to src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx512.cc index 183369d..1fbe5b2 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16_avx512.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once - #include #include #include @@ -23,10 +21,9 @@ namespace zvec::ailego::DistanceBatch { #if defined(__AVX512FP16__) - template static std::enable_if_t, void> -compute_one_to_many_avx512fp16_fp16( +compute_one_to_many_inner_product_avx512fp16_fp16( const ailego::Float16 *query, const ailego::Float16 **ptrs, std::array &prefetch_ptrs, size_t dimensionality, float *results) { @@ -82,7 +79,7 @@ compute_one_to_many_avx512fp16_fp16( template static std::enable_if_t, void> -compute_one_to_many_avx512f_fp16( +compute_one_to_many_inner_product_avx512f_fp16( const ailego::Float16 *query, const ailego::Float16 **ptrs, std::array &prefetch_ptrs, size_t dimensionality, float *results) { @@ -122,7 +119,7 @@ compute_one_to_many_avx512f_fp16( } } - if (dim + 16 < dimensionality) { + if (dim + 16 <= dimensionality) { __m512 q = _mm512_cvtph_ps( _mm256_loadu_si256(reinterpret_cast(query + dim))); @@ -143,7 +140,7 @@ compute_one_to_many_avx512f_fp16( _mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(accs[i]), 1))); } - if (dim + 8 < dimensionality) { + if (dim + 8 <= dimensionality) { __m256 q = _mm256_cvtph_ps( _mm_loadu_si128(reinterpret_cast(query + dim))); @@ -166,78 +163,43 @@ compute_one_to_many_avx512f_fp16( } } } + #endif -#if defined(__AVX2__) - -template -static std::enable_if_t, void> -compute_one_to_many_avx2_fp16( +#if defined(__AVX512FP16__) +void compute_one_to_many_inner_product_avx512fp16_fp16_1( const ailego::Float16 *query, const ailego::Float16 **ptrs, - std::array &prefetch_ptrs, - size_t dimensionality, float *results) { - std::array<__m256, dp_batch> accs; - - for (size_t i = 0; i < dp_batch; ++i) { - accs[i] = _mm256_setzero_ps(); - } - - size_t dim = 0; - for (; dim + 16 <= dimensionality; dim += 16) { - __m256i q = - _mm256_loadu_si256(reinterpret_cast(query + dim)); - - __m256 q1 = _mm256_cvtph_ps(_mm256_castsi256_si128(q)); - __m256 q2 = _mm256_cvtph_ps(_mm256_extractf128_si256(q, 1)); - - std::array<__m256, dp_batch> data_regs_1; - std::array<__m256, dp_batch> data_regs_2; - for (size_t i = 0; i < dp_batch; ++i) { - __m256i m = - _mm256_loadu_si256(reinterpret_cast(ptrs[i] + dim)); - - data_regs_1[i] = _mm256_cvtph_ps(_mm256_castsi256_si128(m)); - data_regs_2[i] = _mm256_cvtph_ps(_mm256_extractf128_si256(m, 1)); - } - - if (prefetch_ptrs[0]) { - for (size_t i = 0; i < dp_batch; ++i) { - ailego_prefetch(prefetch_ptrs[i] + dim); - } - } - - for (size_t i = 0; i < dp_batch; ++i) { - accs[i] = _mm256_fmadd_ps(q1, data_regs_1[i], accs[i]); - accs[i] = _mm256_fmadd_ps(q2, data_regs_2[i], accs[i]); - } - } - - if (dim + 8 < dimensionality) { - __m256 q = _mm256_cvtph_ps( - _mm_loadu_si128(reinterpret_cast(query + dim))); - - std::array<__m256, dp_batch> data_regs; - for (size_t i = 0; i < dp_batch; ++i) { - data_regs[i] = _mm256_cvtph_ps( - _mm_loadu_si128(reinterpret_cast(ptrs[i] + dim))); - accs[i] = _mm256_fmadd_ps(q, data_regs[i], accs[i]); - } - - dim += 8; - } - - for (size_t i = 0; i < dp_batch; ++i) { - results[i] = HorizontalAdd_FP32_V256(accs[i]); - } - - for (; dim < dimensionality; ++dim) { - for (size_t i = 0; i < dp_batch; ++i) { - results[i] += (*(query + dim)) * (*(ptrs[i] + dim)); - } - } + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx512fp16_fp16( + query, ptrs, prefetch_ptrs, dim, sums); } +void compute_one_to_many_inner_product_avx512fp16_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx512fp16_fp16( + query, ptrs, prefetch_ptrs, dim, sums); +} #endif +#if defined(__AVX512F__) +void compute_one_to_many_inner_product_avx512f_fp16_1( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx512f_fp16( + query, ptrs, prefetch_ptrs, dim, sums); +} + +void compute_one_to_many_inner_product_avx512f_fp16_12( + const ailego::Float16 *query, const ailego::Float16 **ptrs, + std::array &prefetch_ptrs, size_t dim, + float *sums) { + return compute_one_to_many_inner_product_avx512f_fp16( + query, ptrs, prefetch_ptrs, dim, sums); +} +#endif } // namespace zvec::ailego::DistanceBatch \ No newline at end of file diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl.h b/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc similarity index 85% rename from src/ailego/math_batch/inner_product_distance_batch_impl.h rename to src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc index d15a747..0e54064 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_fp32_avx2.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once - #include #include #include @@ -22,18 +20,6 @@ namespace zvec::ailego::DistanceBatch { -template -static void compute_one_to_many_fallback( - const ValueType *query, const ValueType **ptrs, - std::array &prefetch_ptrs, size_t dim, - float *sums) { - for (size_t j = 0; j < BatchSize; ++j) { - sums[j] = 0.0; - InnerProductMatrix::Compute(ptrs[j], query, dim, sums + j); - ailego_prefetch(&prefetch_ptrs[j]); - } -} - #if defined(__AVX2__) inline float sum4(__m128 v) { @@ -49,7 +35,7 @@ inline __m128 sum_top_bottom_avx(__m256 v) { template static std::enable_if_t, void> -compute_one_to_many_avx2_fp32( +compute_one_to_many_inner_product_avx2_fp32( const ValueType *query, const ValueType **ptrs, std::array &prefetch_ptrs, size_t dimensionality, float *results) { @@ -123,7 +109,21 @@ compute_one_to_many_avx2_fp32( results[i] = -res[i]; } } + +void compute_one_to_many_inner_product_avx2_fp32_1( + const float *query, const float **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx2_fp32( + query, ptrs, prefetch_ptrs, dim, sums); +} + +void compute_one_to_many_inner_product_avx2_fp32_12( + const float *query, const float **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx2_fp32( + query, ptrs, prefetch_ptrs, dim, sums); +} + #endif - } // namespace zvec::ailego::DistanceBatch \ No newline at end of file diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx2.cc b/src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx2.cc new file mode 100644 index 0000000..23d3566 --- /dev/null +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx2.cc @@ -0,0 +1,102 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +namespace zvec::ailego::DistanceBatch { + +#if defined(__AVX2__) + +template +static std::enable_if_t, void> +compute_one_to_many_inner_product_avx2_int8( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dimensionality, + float *results) { + std::vector<__m256i> accs(dp_batch); + for (size_t i = 0; i < dp_batch; ++i) { + accs[i] = _mm256_setzero_si256(); + } + size_t dim = 0; + for (; dim + 32 <= dimensionality; dim += 32) { + __m256i q = _mm256_loadu_si256((const __m256i *)(query + dim)); + std::vector<__m256i> data_regs(dp_batch); + for (size_t i = 0; i < dp_batch; ++i) { + data_regs[i] = _mm256_loadu_si256((const __m256i *)(ptrs[i] + dim)); + } + if (prefetch_ptrs[0]) { + for (size_t i = 0; i < dp_batch; ++i) { + ailego_prefetch(prefetch_ptrs[i] + dim); + } + } + __m256i q_lo = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(q)); + __m256i q_hi = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(q, 1)); + __m256i data_lo[dp_batch]; + __m256i data_hi[dp_batch]; + for (size_t i = 0; i < dp_batch; ++i) { + data_lo[i] = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(data_regs[i])); + data_hi[i] = + _mm256_cvtepi8_epi16(_mm256_extracti128_si256(data_regs[i], 1)); + } + __m256i prod_lo[dp_batch]; + __m256i prod_hi[dp_batch]; + for (size_t i = 0; i < dp_batch; ++i) { + prod_lo[i] = _mm256_madd_epi16(q_lo, data_lo[i]); + prod_hi[i] = _mm256_madd_epi16(q_hi, data_hi[i]); + } + for (size_t i = 0; i < dp_batch; ++i) { + accs[i] = + _mm256_add_epi32(accs[i], _mm256_add_epi32(prod_lo[i], prod_hi[i])); + } + } + std::array temp_results; + for (size_t i = 0; i < dp_batch; ++i) { + __m128i lo = _mm256_castsi256_si128(accs[i]); + __m128i hi = _mm256_extracti128_si256(accs[i], 1); + __m128i sum128 = _mm_add_epi32(lo, hi); + sum128 = _mm_hadd_epi32(sum128, sum128); + sum128 = _mm_hadd_epi32(sum128, sum128); + temp_results[i] = _mm_cvtsi128_si32(sum128); + } + for (; dim < dimensionality; ++dim) { + int8_t q = query[dim]; + for (size_t i = 0; i < dp_batch; ++i) { + temp_results[i] += q * static_cast(ptrs[i][dim]); + } + } + for (size_t i = 0; i < dp_batch; ++i) { + results[i] = static_cast(temp_results[i]); + } +} + +void compute_one_to_many_inner_product_avx2_int8_1( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx2_int8( + query, ptrs, prefetch_ptrs, dim, sums); +} + +void compute_one_to_many_inner_product_avx2_int8_12( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx2_int8( + query, ptrs, prefetch_ptrs, dim, sums); +} + +#endif + +} // namespace zvec::ailego::DistanceBatch \ No newline at end of file diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h b/src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx512.cc similarity index 68% rename from src/ailego/math_batch/inner_product_distance_batch_impl_int8.h rename to src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx512.cc index 0e23664..1e10583 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_int8_avx512.cc @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once - #include #include #include @@ -23,8 +21,8 @@ namespace zvec::ailego::DistanceBatch { #if defined(__AVX512VNNI__) -static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query, - size_t dim) { +void compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess( + void *query, size_t dim) { const int8_t *input = reinterpret_cast(query); uint8_t *output = reinterpret_cast(query); @@ -48,10 +46,9 @@ static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query, } } - // query is unsigned template -static void compute_one_to_many_avx512_vnni_int8( +static void compute_one_to_many_inner_product_avx512_vnni_int8( const int8_t *query, const int8_t **ptrs, std::array &prefetch_ptrs, size_t dimensionality, float *results) { @@ -153,73 +150,22 @@ static void compute_one_to_many_avx512_vnni_int8( // results[i] = static_cast(temp_results[i]); // } // } -#endif -#if defined(__AVX2__) - -template -static std::enable_if_t, void> -compute_one_to_many_avx2_int8( +void compute_one_to_many_inner_product_avx512_vnni_int8_1( const int8_t *query, const int8_t **ptrs, - std::array &prefetch_ptrs, size_t dimensionality, - float *results) { - std::array<__m256i, dp_batch> accs; - for (size_t i = 0; i < dp_batch; ++i) { - accs[i] = _mm256_setzero_si256(); - } - size_t dim = 0; - for (; dim + 32 <= dimensionality; dim += 32) { - __m256i q = _mm256_loadu_si256((const __m256i *)(query + dim)); - std::array<__m256i, dp_batch> data_regs; - for (size_t i = 0; i < dp_batch; ++i) { - data_regs[i] = _mm256_loadu_si256((const __m256i *)(ptrs[i] + dim)); - } - if (prefetch_ptrs[0]) { - for (size_t i = 0; i < dp_batch; ++i) { - ailego_prefetch(prefetch_ptrs[i] + dim); - } - } - __m256i q_lo = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(q)); - __m256i q_hi = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(q, 1)); - std::array<__m256i, dp_batch> data_lo; - std::array<__m256i, dp_batch> data_hi; - for (size_t i = 0; i < dp_batch; ++i) { - data_lo[i] = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(data_regs[i])); - data_hi[i] = - _mm256_cvtepi8_epi16(_mm256_extracti128_si256(data_regs[i], 1)); - } - std::array<__m256i, dp_batch> prod_lo; - std::array<__m256i, dp_batch> prod_hi; - for (size_t i = 0; i < dp_batch; ++i) { - prod_lo[i] = _mm256_madd_epi16(q_lo, data_lo[i]); - prod_hi[i] = _mm256_madd_epi16(q_hi, data_hi[i]); - } - for (size_t i = 0; i < dp_batch; ++i) { - accs[i] = - _mm256_add_epi32(accs[i], _mm256_add_epi32(prod_lo[i], prod_hi[i])); - } - } - std::array temp_results; - for (size_t i = 0; i < dp_batch; ++i) { - __m128i lo = _mm256_castsi256_si128(accs[i]); - __m128i hi = _mm256_extracti128_si256(accs[i], 1); - __m128i sum128 = _mm_add_epi32(lo, hi); - sum128 = _mm_hadd_epi32(sum128, sum128); - sum128 = _mm_hadd_epi32(sum128, sum128); - temp_results[i] = _mm_cvtsi128_si32(sum128); - } - for (; dim < dimensionality; ++dim) { - int8_t q = query[dim]; - for (size_t i = 0; i < dp_batch; ++i) { - temp_results[i] += q * static_cast(ptrs[i][dim]); - } - } - for (size_t i = 0; i < dp_batch; ++i) { - results[i] = static_cast(temp_results[i]); - } + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx512_vnni_int8<1>( + query, ptrs, prefetch_ptrs, dim, sums); } +void compute_one_to_many_inner_product_avx512_vnni_int8_12( + const int8_t *query, const int8_t **ptrs, + std::array &prefetch_ptrs, size_t dim, float *sums) { + return compute_one_to_many_inner_product_avx512_vnni_int8<12>( + query, ptrs, prefetch_ptrs, dim, sums); +} + + #endif - } // namespace zvec::ailego::DistanceBatch \ No newline at end of file