refactor/march based reorganization (#193)
* add for march compatible test * fix: remove MxN * fix: add macro scope * fix for android ci * refactor: call euclidean via squared euclidean * refactor: add match utility * refactor: comment config out * fix: fix dimension remainder * fix: fix condition & config flags * fix: remove redundant macros * fix: fix condition error * fix: fix macros * fix: remove unnecessary macros * fix: fix remaining code with vector * fix: remove unnecessary codes --------- Co-authored-by: Zefeng Yin <yinzefeng.yzf@alibaba-inc.com> Co-authored-by: xufeihong.xfh <xufeihong.xfh@alibaba-inc.com>
This commit is contained in:
co-authored by
Zefeng Yin
xufeihong.xfh
parent
6b89d97d97
commit
a7bfd6060d
@@ -12,6 +12,13 @@ on:
|
||||
- '**.md'
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-android:
|
||||
# sdkmanager and other Android tools are x86‑only; ARM runners fail with exit code 1
|
||||
@@ -161,6 +168,10 @@ jobs:
|
||||
adb shell getprop ro.product.cpu.abi
|
||||
adb shell getprop ro.product.cpu.abilist
|
||||
|
||||
echo "=== CPU ISA / Instruction Set Support ==="
|
||||
echo "--- /proc/cpuinfo flags ---"
|
||||
adb shell 'cat /proc/cpuinfo | grep -E "^(Features|flags)"'
|
||||
|
||||
echo "Checking binary sizes:"
|
||||
ls -lah examples/c++/build-android-examples-${{ matrix.abi }}/
|
||||
|
||||
|
||||
+44
-28
@@ -13,7 +13,7 @@ option(ENABLE_SAPPHIRERAPIDS "Enable Intel Sapphire Rapids Server CPU microarchi
|
||||
option(ENABLE_EMERALDRAPIDS "Enable Intel Emerald Rapids Server CPU microarchitecture" OFF)
|
||||
option(ENABLE_GRANITERAPIDS "Enable Intel Granite Rapids Server CPU microarchitecture" OFF)
|
||||
|
||||
option(ENABLE_NATIVE "Enable native CPU microarchitecture" ON)
|
||||
option(ENABLE_NATIVE "Enable native CPU microarchitecture" OFF)
|
||||
|
||||
## AMD Microarchitectures
|
||||
option(ENABLE_ZEN1 "Enable AMD Zen+ Family 17h CPU microarchitecture" OFF)
|
||||
@@ -74,41 +74,57 @@ macro(add_arch_flag FLAG VAR_NAME OPTION_NAME)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
function(_detect_armv8_best)
|
||||
set(_arm_flags
|
||||
"armv8.6-a" "armv8.5-a" "armv8.4-a" "armv8.3-a" "armv8.2-a" "armv8.1-a" "armv8-a" "armv8"
|
||||
)
|
||||
foreach(_ver IN LISTS _arm_flags)
|
||||
check_c_compiler_flag("-march=${_ver}" _COMP_SUPP_${_ver})
|
||||
if(_COMP_SUPP_${_ver})
|
||||
_AppendFlags(CMAKE_C_FLAGS "-march=${_ver}")
|
||||
_AppendFlags(CMAKE_CXX_FLAGS "-march=${_ver}")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
endforeach()
|
||||
message(WARNING "No ARMv8 architecture flag supported by compiler.")
|
||||
function(_setup_armv8_march)
|
||||
set(_arch "armv8")
|
||||
check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch})
|
||||
if(_COMP_SUPP_${_arch})
|
||||
_AppendFlags(CMAKE_C_FLAGS "-march=${_arch}")
|
||||
_AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE)
|
||||
return()
|
||||
else()
|
||||
message(WARNING "No ARMv8 march flag supported by compiler.")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(_detect_x86_best)
|
||||
function(_setup_x86_march)
|
||||
set(_arch "x86-64")
|
||||
check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch})
|
||||
if(_COMP_SUPP_${_arch})
|
||||
_AppendFlags(CMAKE_C_FLAGS "-march=${_arch}")
|
||||
_AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE)
|
||||
return()
|
||||
else()
|
||||
message(WARNING "No known x86 march flag supported; falling back to generic.")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
function(setup_compiler_march_for_x86 VAR_NAME_SSE VAR_NAME_AVX2 VAR_NAME_AVX512)
|
||||
#sse
|
||||
set(${VAR_NAME_SSE} "-march=corei7" PARENT_SCOPE)
|
||||
|
||||
#avx 2
|
||||
set(${VAR_NAME_AVX2} "-march=core-avx2" PARENT_SCOPE)
|
||||
|
||||
#avx512
|
||||
set(_x86_flags
|
||||
"graniterapids" "emeraldrapids" "sapphirerapids"
|
||||
"skylake-avx512" "skylake"
|
||||
"broadwell" "haswell" "sandybridge" "nehalem"
|
||||
"znver3" "znver2" "znver1"
|
||||
"graniterapids" "emeraldrapids" "sapphirerapids" "skylake-avx512"
|
||||
)
|
||||
foreach(_arch IN LISTS _x86_flags)
|
||||
check_c_compiler_flag("-march=${_arch}" _COMP_SUPP_${_arch})
|
||||
if(_COMP_SUPP_${_arch})
|
||||
_AppendFlags(CMAKE_C_FLAGS "-march=${_arch}")
|
||||
_AppendFlags(CMAKE_CXX_FLAGS "-march=${_arch}")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}" PARENT_SCOPE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE)
|
||||
set(${VAR_NAME_AVX512} "-march=${_arch}" PARENT_SCOPE)
|
||||
return()
|
||||
endif()
|
||||
endforeach()
|
||||
message(WARNING "No known x86 microarchitecture flag supported; falling back to generic.")
|
||||
|
||||
|
||||
set(${VAR_NAME_AVX512} "-march=core-avx2" PARENT_SCOPE)
|
||||
message(WARNING "No known avx512 microarchitecture flag found. Set up as core-avx2")
|
||||
|
||||
endfunction()
|
||||
|
||||
if(MSVC)
|
||||
@@ -206,9 +222,9 @@ else()
|
||||
# AUTO DETECT
|
||||
# Heuristic: detect host architecture and probe appropriate flags
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
|
||||
_detect_armv8_best()
|
||||
_setup_armv8_march()
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64")
|
||||
_detect_x86_best()
|
||||
_setup_x86_march()
|
||||
else()
|
||||
message(WARNING "Unknown host architecture: ${CMAKE_SYSTEM_PROCESSOR}; no -march= set.")
|
||||
endif()
|
||||
|
||||
@@ -124,6 +124,7 @@ sdist.include = [
|
||||
[tool.scikit-build.cmake.define]
|
||||
BUILD_TOOLS = "OFF"
|
||||
BUILD_PYTHON_BINDINGS = "ON"
|
||||
#CMAKE_VERBOSE_MAKEFILE = "ON"
|
||||
|
||||
# Setuptools config for test pypi
|
||||
[tool.setuptools_scm]
|
||||
|
||||
@@ -18,6 +18,89 @@ if(UNIX AND NOT APPLE)
|
||||
list(APPEND EXTRA_LIBS ${LIB_RT})
|
||||
endif()
|
||||
|
||||
if(NOT ANDROID)
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|i686|i386|x64")
|
||||
setup_compiler_march_for_x86(MATH_MARCH_FLAG_SSE MATH_MARCH_FLAG_AVX2 MATH_MARCH_FLAG_AVX512)
|
||||
message(STATUS "best compiler march, sse: " ${MATH_MARCH_FLAG_SSE} ", avx2: " ${MATH_MARCH_FLAG_AVX2} ", avx512: " ${MATH_MARCH_FLAG_AVX512})
|
||||
|
||||
file(GLOB_RECURSE MATH_FILES_SSE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_sse.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_sse.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_sse.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_sse.c
|
||||
)
|
||||
|
||||
file(GLOB_RECURSE MATH_FILES_AVX2
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx2.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx2.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx2.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx2.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx.c
|
||||
)
|
||||
|
||||
file(GLOB_RECURSE MATH_FILES_AVX512
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx512.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_avx512.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx512.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_avx512.c
|
||||
)
|
||||
|
||||
foreach(MATH_FILE ${MATH_FILES_SSE})
|
||||
set_source_files_properties(
|
||||
${MATH_FILE}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${MATH_MARCH_FLAG_SSE}"
|
||||
)
|
||||
endforeach()
|
||||
|
||||
foreach(MATH_FILE ${MATH_FILES_AVX2})
|
||||
set_source_files_properties(
|
||||
${MATH_FILE}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${MATH_MARCH_FLAG_AVX2}"
|
||||
)
|
||||
endforeach()
|
||||
|
||||
foreach(MATH_FILE ${MATH_FILES_AVX512})
|
||||
set_source_files_properties(
|
||||
${MATH_FILE}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${MATH_MARCH_FLAG_AVX512}"
|
||||
)
|
||||
endforeach()
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
|
||||
# set(CMAKE_CXX_FLAGS "-march=armv8-a")
|
||||
# set(CMAKE_C_FLAGS "-march=armv8-a")
|
||||
set(MATH_MARCH_FLAG_NEON "-march=armv8-a")
|
||||
|
||||
file(GLOB_RECURSE MATH_FILES_NEON
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_dispatch.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_dispatch.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math/*_neon.c
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/math_batch/*_neon.c
|
||||
)
|
||||
|
||||
foreach(MATH_FILE ${MATH_FILES_NEON})
|
||||
set_source_files_properties(
|
||||
${MATH_FILE}
|
||||
PROPERTIES
|
||||
COMPILE_FLAGS "${MATH_MARCH_FLAG_NEON}"
|
||||
)
|
||||
endforeach()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME zvec_ailego STATIC STRICT PACKED
|
||||
SRCS ${ALL_SRCS}
|
||||
@@ -25,4 +108,4 @@ cc_library(
|
||||
Arrow::arrow_static
|
||||
Arrow::parquet_static
|
||||
VERSION "${GIT_SRCS_VER}"
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_FP32_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
float x = m - q; \
|
||||
sum += (x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \
|
||||
xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \
|
||||
ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP32_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float32x4_t v_d = vsubq_f32(v_m, v_q); \
|
||||
v_sum = vfmaq_f32(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_FP16_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
float x = m - q; \
|
||||
sum += (x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP16_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float16x8_t v_d = vsubq_f16(v_m, v_q); \
|
||||
v_sum = vfmaq_f16(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX512)
|
||||
#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
{ \
|
||||
__m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_INT4_GENERAL(m, q, sum) \
|
||||
sum += Int4SquaredDiffTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \
|
||||
Int4SquaredDiffTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)];
|
||||
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0xf0f0f0f0);
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
//! Compute the square root of value (SSE)
|
||||
#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v))
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0xf0f0f0f0);
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_INT4_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_m), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_q), 4), MASK_INT4_SSE); \
|
||||
xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs)), \
|
||||
4); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_lhs = _mm_and_si128((xmm_m), MASK_INT4_SSE); \
|
||||
xmm_rhs = _mm_and_si128((xmm_q), MASK_INT4_SSE); \
|
||||
xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs)), \
|
||||
4); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_lhs), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs_0 = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_rhs), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_lhs_1 = _mm_and_si128((xmm_lhs), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs_1 = _mm_and_si128((xmm_rhs), MASK_INT4_SSE); \
|
||||
xmm_lhs_0 = \
|
||||
_mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), \
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0)), \
|
||||
4); \
|
||||
xmm_rhs_0 = \
|
||||
_mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), \
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1)), \
|
||||
4); \
|
||||
xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs_0, xmm_lhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_rhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_rhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_rhs_0), xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_INT4_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_m), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_q), 4), MASK_INT4_AVX); \
|
||||
ymm_lhs = \
|
||||
_mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs)), \
|
||||
4); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_lhs = _mm256_and_si256((ymm_m), MASK_INT4_AVX); \
|
||||
ymm_rhs = _mm256_and_si256((ymm_q), MASK_INT4_AVX); \
|
||||
ymm_lhs = \
|
||||
_mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs)), \
|
||||
4); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_lhs), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs_0 = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_rhs), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_lhs_1 = _mm256_and_si256((ymm_lhs), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs_1 = _mm256_and_si256((ymm_rhs), MASK_INT4_AVX); \
|
||||
ymm_lhs_0 = _mm256_srli_epi32( \
|
||||
_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), \
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)), \
|
||||
4); \
|
||||
ymm_rhs_0 = _mm256_srli_epi32( \
|
||||
_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), \
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)), \
|
||||
4); \
|
||||
ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs_0, ymm_lhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_rhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_rhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_sum = \
|
||||
_mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_rhs_0), ymm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_INT8_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
int32_t x = m - q; \
|
||||
sum += static_cast<float>(x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_INT8_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_m), \
|
||||
_mm_sign_epi8(xmm_m, xmm_m)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \
|
||||
_mm_sign_epi8(xmm_q, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_sum = _mm_sub_epi32( \
|
||||
xmm_sum, \
|
||||
_mm_slli_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \
|
||||
_mm_sign_epi8(xmm_m, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
1)); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_INT8_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_m), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_m)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_q, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_sum = _mm256_sub_epi32( \
|
||||
ymm_sum, _mm256_slli_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
1)); \
|
||||
}
|
||||
|
||||
//! Compute the square root of value (AVX)
|
||||
#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v))
|
||||
|
||||
//! Compute the square root of value (AVX512)
|
||||
#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v))
|
||||
|
||||
#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE
|
||||
#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX
|
||||
|
||||
#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512
|
||||
#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL
|
||||
|
||||
#define ACCUM_FP16_STEP_NEON SSD_FP16_NEON
|
||||
#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON
|
||||
|
||||
#define ACCUM_INT4_STEP_SSE SSD_INT4_SSE
|
||||
#define ACCUM_INT4_STEP_AVX SSD_INT4_AVX
|
||||
#define ACCUM_INT8_STEP_SSE SSD_INT8_SSE
|
||||
#define ACCUM_INT8_STEP_AVX SSD_INT8_AVX
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Four-bits Convert Table
|
||||
static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1};
|
||||
|
||||
#define NEGZEROS_FP32_SSE _mm_set1_ps(-0.0f)
|
||||
#define MASK_INT4_SSE _mm_set1_epi32(0x0f0f0f0f)
|
||||
#define ONES_INT16_SSE _mm_set1_epi32(0x00010001)
|
||||
#define INT4_LOOKUP_SSE _mm_load_si128((const __m128i *)Int4ConvertTable)
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX__)
|
||||
// #define NEGZEROS_FP32_AVX _mm256_set1_ps(-0.0f)
|
||||
#define MASK_INT4_AVX _mm256_set1_epi32(0x0f0f0f0f)
|
||||
#define ONES_INT16_AVX _mm256_set1_epi32(0x00010001)
|
||||
#define INT4_LOOKUP_AVX _mm256_load_si256((const __m256i *)Int4ConvertTable)
|
||||
#endif // __AVX__
|
||||
|
||||
#if defined(__AVX512F__) && !defined(__AVX512DQ__)
|
||||
#define _mm512_xor_ps(a, b) \
|
||||
_mm512_castsi512_ps( \
|
||||
_mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b)))
|
||||
#endif // __AVX512DQ__
|
||||
|
||||
//! Reverse sign of value (GENERAL)
|
||||
#define NEGATE_FP32_GENERAL(v) -(v)
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_FP32_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
xmm_sum = _mm_fmadd_ps(xmm_m, xmm_q, xmm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_FP32_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
ymm_sum = _mm256_fmadd_ps(ymm_m, ymm_q, ymm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX512)
|
||||
#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX512FP16)
|
||||
#define FMA_FP16_AVX512FP16(zmm_m, zmm_q, zmm_sum) \
|
||||
zmm_sum = _mm512_fmadd_ph(zmm_m, zmm_q, zmm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_FP16_GENERAL(m, q, sum) sum += (m * q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_FP32_GENERAL(m, q, sum) sum += (m * q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (NEON)
|
||||
#define FMA_FP16_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f16(v_sum, v_m, v_q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (NEON)
|
||||
#define FMA_FP32_NEON(v_m, v_q, v_sum) v_sum = vfmaq_f32(v_sum, v_m, v_q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT4_GENERAL(m, q, sum) \
|
||||
sum += Int4MulTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \
|
||||
Int4MulTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)];
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast<float>(m * q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT8_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16( \
|
||||
_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), _mm_sign_epi8(xmm_m, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT8_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT4_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128((xmm_m), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128((xmm_q), MASK_INT4_SSE)); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_lhs = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_m), 4), MASK_INT4_SSE)); \
|
||||
xmm_rhs = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_q), 4), MASK_INT4_SSE)); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT4_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_m), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_q), MASK_INT4_AVX)); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_lhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_m), 4), MASK_INT4_AVX)); \
|
||||
ymm_rhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_q), 4), MASK_INT4_AVX)); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_lhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); \
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); \
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); \
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); \
|
||||
xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_lhs_1 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_lhs_1), xmm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_lhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); \
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); \
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); \
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); \
|
||||
ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_lhs_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_sum = \
|
||||
_mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_lhs_1), ymm_sum); \
|
||||
}
|
||||
|
||||
#define ACCUM_FP16_STEP_GENERAL FMA_FP16_GENERAL
|
||||
#define ACCUM_FP16_STEP_NEON FMA_FP16_NEON
|
||||
|
||||
#define ACCUM_FP32_STEP_SSE FMA_FP32_SSE
|
||||
#define ACCUM_FP32_STEP_AVX FMA_FP32_AVX
|
||||
#define ACCUM_FP32_STEP_AVX512 FMA_FP32_AVX512
|
||||
#define ACCUM_FP32_STEP_NEON FMA_FP32_NEON
|
||||
|
||||
#define ACCUM_INT4_STEP_SSE FMA_INT4_SSE
|
||||
#define ACCUM_INT4_STEP_AVX FMA_INT4_AVX
|
||||
|
||||
#define ACCUM_INT8_STEP_SSE FMA_INT8_SSE
|
||||
#define ACCUM_INT8_STEP_AVX FMA_INT8_AVX
|
||||
@@ -0,0 +1,160 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX512)
|
||||
#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum);
|
||||
|
||||
#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \
|
||||
zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask);
|
||||
|
||||
#define HorizontalAdd_FP16_NEON(v) \
|
||||
vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(v)), vcvt_high_f32_f16(v)))
|
||||
|
||||
#define HorizontalAdd_FP32_V512_TO_V256(zmm) \
|
||||
_mm256_add_ps( \
|
||||
_mm512_castps512_ps256(zmm), \
|
||||
_mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(zmm), 1)))
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX, FP16)
|
||||
#define FMA_FP16_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
float v1 = lhs; \
|
||||
float v2 = rhs; \
|
||||
sum += v1 * v2; \
|
||||
norm1 += v1 * v1; \
|
||||
norm2 += v2 * v2; \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_FP32_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += (lhs) * (rhs); \
|
||||
norm1 += (lhs) * (lhs); \
|
||||
norm2 += (rhs) * (rhs); \
|
||||
}
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Four-bits Convert Table
|
||||
static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1};
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f);
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
static const __m128i INT4_LOOKUP_SSE =
|
||||
_mm_load_si128((const __m128i *)Int4ConvertTable);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f);
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
static const __m256i INT4_LOOKUP_AVX =
|
||||
_mm256_load_si256((const __m256i *)Int4ConvertTable);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT4_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += Int4MulTable[(((lhs) << 4) & 0xf0) | (((rhs) >> 0) & 0xf)] + \
|
||||
Int4MulTable[(((lhs) >> 0) & 0xf0) | (((rhs) >> 4) & 0xf)]; \
|
||||
norm1 += static_cast<float>( \
|
||||
((int8_t)((lhs) << 4) >> 4) * ((int8_t)((lhs) << 4) >> 4) + \
|
||||
((int8_t)((lhs) & 0xf0) >> 4) * ((int8_t)((lhs) & 0xf0) >> 4)); \
|
||||
norm2 += static_cast<float>( \
|
||||
((int8_t)((rhs) << 4) >> 4) * ((int8_t)((rhs) << 4) >> 4) + \
|
||||
((int8_t)((rhs) & 0xf0) >> 4) * ((int8_t)((rhs) & 0xf0) >> 4)); \
|
||||
}
|
||||
|
||||
|
||||
//! Compute the distance between matrix and query (SSE)
|
||||
#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum_0, xmm_sum_norm1, \
|
||||
xmm_sum_norm2) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_lhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); \
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); \
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); \
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum_0); \
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); \
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT8_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += static_cast<float>(lhs * rhs); \
|
||||
norm1 += static_cast<float>(lhs * lhs); \
|
||||
norm2 += static_cast<float>(rhs * rhs); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum)
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum)
|
||||
|
||||
#define FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_set_m128i( \
|
||||
_mm_setzero_si128(), \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE)), \
|
||||
ymm_sum)
|
||||
|
||||
//! Compute the distance between matrix and query (AVX)
|
||||
#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, \
|
||||
ymm_sum_norm1, ymm_sum_norm2) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_lhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); \
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); \
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); \
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); \
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); \
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); \
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,615 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE
|
||||
#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX
|
||||
#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512
|
||||
#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON
|
||||
#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL
|
||||
#define ACCUM_FP16_STEP_NEON SSD_FP16_NEON
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \
|
||||
xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \
|
||||
ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX512)
|
||||
#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
{ \
|
||||
__m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_FP16_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
float x = m - q; \
|
||||
sum += (x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP16_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float16x8_t v_d = vsubq_f16(v_m, v_q); \
|
||||
v_sum = vfmaq_f16(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP32_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float32x4_t v_d = vsubq_f32(v_m, v_q); \
|
||||
v_sum = vfmaq_f32(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
#if (defined(__F16C__) && defined(__AVX__)) || \
|
||||
(defined(__ARM_NEON) && defined(__aarch64__))
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs,
|
||||
const Float16 *rhs,
|
||||
size_t size) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
__m512h zmm_sum_0 = _mm512_setzero_ph();
|
||||
__m512h zmm_sum_1 = _mm512_setzero_ph();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m512h zmm_d_0 =
|
||||
_mm512_sub_ph(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0));
|
||||
__m512h zmm_d_1 =
|
||||
_mm512_sub_ph(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m512h zmm_d = _mm512_sub_ph(_mm512_load_ph(lhs), _mm512_load_ph(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m512h zmm_d_0 =
|
||||
_mm512_sub_ph(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0));
|
||||
__m512h zmm_d_1 =
|
||||
_mm512_sub_ph(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m512h zmm_d = _mm512_sub_ph(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1);
|
||||
__m512i zmm_undefined = _mm512_undefined_epi32();
|
||||
__m512h zmm_undefined_ph = _mm512_undefined_ph();
|
||||
__m512h zmm_d = _mm512_mask_sub_ph(
|
||||
zmm_undefined_ph, mask,
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)),
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs)));
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ph(zmm_d, zmm_d, zmm_sum_0, mask);
|
||||
}
|
||||
|
||||
return HorizontalAdd_FP16_V512(zmm_sum_0);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, )
|
||||
#else
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
*out = SquaredEuclideanDistanceAVX512FP16(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, )
|
||||
#endif //__ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, std::sqrt)
|
||||
#else
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
*out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim));
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt)
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, std::sqrt)
|
||||
#endif //__ARM_NEON
|
||||
}
|
||||
|
||||
#if !defined(__ARM_NEON)
|
||||
//! Compute the distance between matrix and query (FP16, M=2, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_2X1_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=2, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_2X2_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_4X1_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_4X2_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_4X4_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_8X1_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_8X2_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_8X4_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
ACCUM_FP16_8X8_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X1_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
ACCUM_FP16_16X1_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X2_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
ACCUM_FP16_16X2_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X4_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X4_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X8_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X8_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 16, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X16_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X16_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X1_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X1_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X2_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X2_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X4_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X4_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X8_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X8_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X16_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X16_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=32)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 32, 32>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X32_AVX512(m, q, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X32_AVX(m, q, dim, out, )
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=2, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_2X1_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=2, N=2)
|
||||
void EuclideanDistanceMatrix<Float16, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_2X2_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_4X1_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=2)
|
||||
void EuclideanDistanceMatrix<Float16, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_4X2_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=4, N=4)
|
||||
void EuclideanDistanceMatrix<Float16, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_4X4_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_8X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=2)
|
||||
void EuclideanDistanceMatrix<Float16, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_8X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=4)
|
||||
void EuclideanDistanceMatrix<Float16, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_8X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=8, N=8)
|
||||
void EuclideanDistanceMatrix<Float16, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
ACCUM_FP16_8X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=2)
|
||||
void EuclideanDistanceMatrix<Float16, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=4)
|
||||
void EuclideanDistanceMatrix<Float16, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=8)
|
||||
void EuclideanDistanceMatrix<Float16, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=16, N=16)
|
||||
void EuclideanDistanceMatrix<Float16, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=2)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=4)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=8)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=16)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=32, N=32)
|
||||
void EuclideanDistanceMatrix<Float16, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
return;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
}
|
||||
#endif // !__ARM_NEON
|
||||
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__)
|
||||
|
||||
void SquaredEuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out) {
|
||||
ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, )
|
||||
}
|
||||
|
||||
//! EuclideanDistance
|
||||
void EuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out) {
|
||||
ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, std::sqrt)
|
||||
}
|
||||
|
||||
#endif // __AVX__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
__m512h zmm_sum_0 = _mm512_setzero_ph();
|
||||
__m512h zmm_sum_1 = _mm512_setzero_ph();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m512h zmm_d_0 =
|
||||
_mm512_sub_ph(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0));
|
||||
__m512h zmm_d_1 =
|
||||
_mm512_sub_ph(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m512h zmm_d = _mm512_sub_ph(_mm512_load_ph(lhs), _mm512_load_ph(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m512h zmm_d_0 =
|
||||
_mm512_sub_ph(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0));
|
||||
__m512h zmm_d_1 =
|
||||
_mm512_sub_ph(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ph(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m512h zmm_d = _mm512_sub_ph(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ph(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1);
|
||||
__m512i zmm_undefined = _mm512_undefined_epi32();
|
||||
__m512h zmm_undefined_ph = _mm512_undefined_ph();
|
||||
__m512h zmm_d = _mm512_mask_sub_ph(
|
||||
zmm_undefined_ph, mask,
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)),
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs)));
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ph(zmm_d, zmm_d, zmm_sum_0, mask);
|
||||
}
|
||||
|
||||
return HorizontalAdd_FP16_V512(zmm_sum_0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void SquaredEuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out) {
|
||||
ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, )
|
||||
}
|
||||
|
||||
//! EuclideanDistance
|
||||
void EuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out) {
|
||||
ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, std::sqrt)
|
||||
}
|
||||
|
||||
#endif
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
void SquaredEuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out);
|
||||
void EuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void SquaredEuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out);
|
||||
|
||||
void EuclideanDistanceAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
void SquaredEuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out);
|
||||
void EuclideanDistanceAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out);
|
||||
#endif
|
||||
|
||||
#if (defined(__F16C__) && defined(__AVX__)) || \
|
||||
(defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
SquaredEuclideanDistanceNEON(m, q, dim, out);
|
||||
#else
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
*out = SquaredEuclideanDistanceAVX512FP16(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
SquaredEuclideanDistanceAVX512(m, q, dim, out);
|
||||
// ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
SquaredEuclideanDistanceAVX(m, q, dim, out);
|
||||
// ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, )
|
||||
#endif //__ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
SquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(m, q, dim, out);
|
||||
*out = std::sqrt(*out);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
void SquaredEuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out) {
|
||||
ACCUM_FP16_1X1_NEON(lhs, rhs, size, out, 0ull, )
|
||||
}
|
||||
|
||||
void EuclideanDistanceNEON(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out) {
|
||||
ACCUM_FP16_1X1_NEON(lhs, rhs, size, out, 0ull, std::sqrt)
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE
|
||||
#define ACCUM_FP16_STEP_GENERAL SSD_FP16_GENERAL
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \
|
||||
xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_FP16_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
float x = m - q; \
|
||||
sum += (x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP16_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float16x8_t v_d = vsubq_f16(v_m, v_q); \
|
||||
v_sum = vfmaq_f16(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP32_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float32x4_t v_d = vsubq_f32(v_m, v_q); \
|
||||
v_sum = vfmaq_f32(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,930 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_FP32_STEP_SSE SSD_FP32_SSE
|
||||
#define ACCUM_FP32_STEP_AVX SSD_FP32_AVX
|
||||
#define ACCUM_FP32_STEP_AVX512 SSD_FP32_AVX512
|
||||
#define ACCUM_FP32_STEP_NEON SSD_FP32_NEON
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_FP32_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
float x = m - q; \
|
||||
sum += (x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_FP32_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128 xmm_d = _mm_sub_ps(xmm_m, xmm_q); \
|
||||
xmm_sum = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_FP32_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256 ymm_d = _mm256_sub_ps(ymm_m, ymm_q); \
|
||||
ymm_sum = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX512)
|
||||
#define SSD_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
{ \
|
||||
__m512 zmm_d = _mm512_sub_ps(zmm_m, zmm_q); \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (NEON)
|
||||
#define SSD_FP32_NEON(v_m, v_q, v_sum) \
|
||||
{ \
|
||||
float32x4_t v_d = vsubq_f32(v_m, v_q); \
|
||||
v_sum = vfmaq_f32(v_sum, v_d, v_d); \
|
||||
}
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceNEON(const float *lhs,
|
||||
const float *rhs,
|
||||
size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float32x4_t v_d_0 = vsubq_f32(vld1q_f32(lhs + 0), vld1q_f32(rhs + 0));
|
||||
float32x4_t v_d_1 = vsubq_f32(vld1q_f32(lhs + 4), vld1q_f32(rhs + 4));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_d_0, v_d_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_d_1, v_d_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_d = vsubq_f32(vld1q_f32(lhs), vld1q_f32(rhs));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_d, v_d);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __ARM_NEON
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceSSE(const float *lhs,
|
||||
const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
__m128 xmm_sum_0 = _mm_setzero_ps();
|
||||
__m128 xmm_sum_1 = _mm_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_d_0 = _mm_sub_ps(_mm_load_ps(lhs + 0), _mm_load_ps(rhs + 0));
|
||||
__m128 xmm_d_1 = _mm_sub_ps(_mm_load_ps(lhs + 4), _mm_load_ps(rhs + 4));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_d = _mm_sub_ps(_mm_load_ps(lhs), _mm_load_ps(rhs));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_d_0 = _mm_sub_ps(_mm_loadu_ps(lhs + 0), _mm_loadu_ps(rhs + 0));
|
||||
__m128 xmm_d_1 = _mm_sub_ps(_mm_loadu_ps(lhs + 4), _mm_loadu_ps(rhs + 4));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_d = _mm_sub_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE__
|
||||
|
||||
#if defined(__AVX__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceAVX(const float *lhs,
|
||||
const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_d_0 =
|
||||
_mm256_sub_ps(_mm256_load_ps(lhs + 0), _mm256_load_ps(rhs + 0));
|
||||
__m256 ymm_d_1 =
|
||||
_mm256_sub_ps(_mm256_load_ps(lhs + 8), _mm256_load_ps(rhs + 8));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_d = _mm256_sub_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_d_0 =
|
||||
_mm256_sub_ps(_mm256_loadu_ps(lhs + 0), _mm256_loadu_ps(rhs + 0));
|
||||
__m256 ymm_d_1 =
|
||||
_mm256_sub_ps(_mm256_loadu_ps(lhs + 8), _mm256_loadu_ps(rhs + 8));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_d = _mm256_sub_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
SSD_FP32_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_FP32_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_FP32_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_FP32_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceAVX512(const float *lhs,
|
||||
const float *rhs,
|
||||
size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_d_0 =
|
||||
_mm512_sub_ps(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0));
|
||||
__m512 zmm_d_1 =
|
||||
_mm512_sub_ps(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_d = _mm512_sub_ps(_mm512_load_ps(lhs), _mm512_load_ps(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_d_0 =
|
||||
_mm512_sub_ps(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0));
|
||||
__m512 zmm_d_1 =
|
||||
_mm512_sub_ps(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_d = _mm512_sub_ps(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1);
|
||||
__m512 zmm_undefined = _mm512_undefined_ps();
|
||||
__m512 zmm_d = _mm512_mask_sub_ps(
|
||||
zmm_undefined, mask, _mm512_mask_loadu_ps(zmm_undefined, mask, lhs),
|
||||
_mm512_mask_loadu_ps(zmm_undefined, mask, rhs));
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ps(zmm_d, zmm_d, zmm_sum_0, mask);
|
||||
}
|
||||
return HorizontalAdd_FP32_V512(zmm_sum_0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__) || defined(__ARM_NEON)
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = SquaredEuclideanDistanceNEON(m, q, dim);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
if (dim > 15) {
|
||||
*out = SquaredEuclideanDistanceAVX512(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
if (dim > 7) {
|
||||
*out = SquaredEuclideanDistanceAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim);
|
||||
#endif // __ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=2, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_2X1_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_2X1_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_2X1_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=2, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<float, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_2X2_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_2X2_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_2X2_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X1_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X1_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_4X1_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<float, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X2_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X2_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_4X2_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<float, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X4_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X4_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_4X4_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X1_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X1_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_8X1_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<float, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X2_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X2_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_8X2_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<float, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X4_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X4_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_8X4_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<float, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X8_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X8_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_8X8_SSE(m, q, dim, out, )
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X1_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X1_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X1_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_16X1_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<float, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X2_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X2_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X2_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_16X2_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<float, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X4_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X4_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X4_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_16X4_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<float, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X8_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X8_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X8_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_16X8_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<float, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X16_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X16_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X16_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_16X16_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X1_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X1_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X1_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X1_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X2_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X2_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X2_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X2_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X4_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X4_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X4_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X4_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X8_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X8_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X8_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X8_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X16_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X16_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X16_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X16_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=32)
|
||||
void SquaredEuclideanDistanceMatrix<float, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X32_NEON(m, q, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X32_AVX512(m, q, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X32_AVX(m, q, dim, out, )
|
||||
#else
|
||||
ACCUM_FP32_32X32_SSE(m, q, dim, out, )
|
||||
#endif
|
||||
}
|
||||
#endif // __SSE__ || __ARM_NEON
|
||||
|
||||
#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = std::sqrt(SquaredEuclideanDistanceNEON(m, q, dim));
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
if (dim > 15) {
|
||||
*out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim));
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
if (dim > 7) {
|
||||
*out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim));
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX__
|
||||
*out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim));
|
||||
#endif // __ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=2, N=1)
|
||||
void EuclideanDistanceMatrix<float, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_2X1_NEON(m, q, dim, out, vsqrt_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_2X1_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_2X1_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=2, N=2)
|
||||
void EuclideanDistanceMatrix<float, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_2X2_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_2X2_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_2X2_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=1)
|
||||
void EuclideanDistanceMatrix<float, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X1_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X1_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_4X1_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=2)
|
||||
void EuclideanDistanceMatrix<float, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X2_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X2_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_4X2_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=4, N=4)
|
||||
void EuclideanDistanceMatrix<float, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_4X4_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_4X4_AVX(m, q, dim, out, _mm_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_4X4_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=1)
|
||||
void EuclideanDistanceMatrix<float, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X1_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_8X1_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=2)
|
||||
void EuclideanDistanceMatrix<float, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X2_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_8X2_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=4)
|
||||
void EuclideanDistanceMatrix<float, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X4_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_8X4_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=8, N=8)
|
||||
void EuclideanDistanceMatrix<float, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_8X8_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_8X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_8X8_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif // __AVX__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=1)
|
||||
void EuclideanDistanceMatrix<float, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X1_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_16X1_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=2)
|
||||
void EuclideanDistanceMatrix<float, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X2_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_16X2_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=4)
|
||||
void EuclideanDistanceMatrix<float, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X4_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_16X4_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=8)
|
||||
void EuclideanDistanceMatrix<float, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X8_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_16X8_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=16, N=16)
|
||||
void EuclideanDistanceMatrix<float, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_16X16_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_16X16_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=1)
|
||||
void EuclideanDistanceMatrix<float, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X1_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X1_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=2)
|
||||
void EuclideanDistanceMatrix<float, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X2_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X2_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=4)
|
||||
void EuclideanDistanceMatrix<float, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X4_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X4_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=8)
|
||||
void EuclideanDistanceMatrix<float, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X8_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X8_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=16)
|
||||
void EuclideanDistanceMatrix<float, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X16_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X16_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=32, N=32)
|
||||
void EuclideanDistanceMatrix<float, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
ACCUM_FP32_32X32_NEON(m, q, dim, out, vsqrtq_f32)
|
||||
#elif defined(__AVX512F__)
|
||||
ACCUM_FP32_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps)
|
||||
#elif defined(__AVX__)
|
||||
ACCUM_FP32_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps)
|
||||
#else
|
||||
ACCUM_FP32_32X32_SSE(m, q, dim, out, _mm_sqrt_ps)
|
||||
#endif
|
||||
}
|
||||
#endif // __SSE__ || __ARM_NEON && __aarch64__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,94 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__)
|
||||
float SquaredEuclideanDistanceAVX(const float *lhs, const float *rhs,
|
||||
size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_d_0 =
|
||||
_mm256_sub_ps(_mm256_load_ps(lhs + 0), _mm256_load_ps(rhs + 0));
|
||||
__m256 ymm_d_1 =
|
||||
_mm256_sub_ps(_mm256_load_ps(lhs + 8), _mm256_load_ps(rhs + 8));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_d = _mm256_sub_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_d_0 =
|
||||
_mm256_sub_ps(_mm256_loadu_ps(lhs + 0), _mm256_loadu_ps(rhs + 0));
|
||||
__m256 ymm_d_1 =
|
||||
_mm256_sub_ps(_mm256_loadu_ps(lhs + 8), _mm256_loadu_ps(rhs + 8));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d_0, ymm_d_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_d_1, ymm_d_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_d = _mm256_sub_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_d, ymm_d, ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
SSD_FP32_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_FP32_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_FP32_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_FP32_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __AVX__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceAVX512(const float *lhs, const float *rhs,
|
||||
size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_d_0 =
|
||||
_mm512_sub_ps(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0));
|
||||
__m512 zmm_d_1 =
|
||||
_mm512_sub_ps(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_d = _mm512_sub_ps(_mm512_load_ps(lhs), _mm512_load_ps(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_d_0 =
|
||||
_mm512_sub_ps(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0));
|
||||
__m512 zmm_d_1 =
|
||||
_mm512_sub_ps(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d_0, zmm_d_0, zmm_sum_0);
|
||||
zmm_sum_1 = _mm512_fmadd_ps(zmm_d_1, zmm_d_1, zmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_d = _mm512_sub_ps(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs));
|
||||
zmm_sum_0 = _mm512_fmadd_ps(zmm_d, zmm_d, zmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1);
|
||||
__m512 zmm_undefined = _mm512_undefined_ps();
|
||||
__m512 zmm_d = _mm512_mask_sub_ps(
|
||||
zmm_undefined, mask, _mm512_mask_loadu_ps(zmm_undefined, mask, lhs),
|
||||
_mm512_mask_loadu_ps(zmm_undefined, mask, rhs));
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ps(zmm_d, zmm_d, zmm_sum_0, mask);
|
||||
}
|
||||
return HorizontalAdd_FP32_V512(zmm_sum_0);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
void SquaredEuclideanDistanceNEON(const float *lhs, const float *rhs,
|
||||
size_t size, float *out);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
float SquaredEuclideanDistanceAVX512(const float *lhs, const float *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceAVX512(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
float SquaredEuclideanDistanceAVX(const float *lhs, const float *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceAVX(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__)
|
||||
float SquaredEuclideanDistanceSSE(const float *lhs, const float *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceSSE(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
//-----------------------------------------------------------
|
||||
// SquaredEuclideanDistance
|
||||
//-----------------------------------------------------------
|
||||
#if defined(__SSE__) || defined(__ARM_NEON)
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
SquaredEuclideanDistanceNEON(m, q, dim, out);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
if (dim > 15) {
|
||||
*out = SquaredEuclideanDistanceAVX512(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
if (dim > 7) {
|
||||
*out = SquaredEuclideanDistanceAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim);
|
||||
#endif // __ARM_NEON
|
||||
}
|
||||
#endif // __SSE__ || __ARM_NEON
|
||||
|
||||
|
||||
//-----------------------------------------------------------
|
||||
// EuclideanDistance
|
||||
//-----------------------------------------------------------
|
||||
#if defined(__SSE__) || (defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(m, q, dim, out);
|
||||
*out = std::sqrt(*out);
|
||||
}
|
||||
#endif // __SSE__ || __ARM_NEON && __aarch64__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Squared Euclidean Distance
|
||||
void SquaredEuclideanDistanceNEON(const float *lhs, const float *rhs,
|
||||
size_t size, float *out) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float32x4_t v_d_0 = vsubq_f32(vld1q_f32(lhs + 0), vld1q_f32(rhs + 0));
|
||||
float32x4_t v_d_1 = vsubq_f32(vld1q_f32(lhs + 4), vld1q_f32(rhs + 4));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_d_0, v_d_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_d_1, v_d_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_d = vsubq_f32(vld1q_f32(lhs), vld1q_f32(rhs));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_d, v_d);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
*out = result;
|
||||
}
|
||||
|
||||
#endif // __ARM_NEON
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE__)
|
||||
float SquaredEuclideanDistanceSSE(const float *lhs, const float *rhs,
|
||||
size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
__m128 xmm_sum_0 = _mm_setzero_ps();
|
||||
__m128 xmm_sum_1 = _mm_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_d_0 = _mm_sub_ps(_mm_load_ps(lhs + 0), _mm_load_ps(rhs + 0));
|
||||
__m128 xmm_d_1 = _mm_sub_ps(_mm_load_ps(lhs + 4), _mm_load_ps(rhs + 4));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_d = _mm_sub_ps(_mm_load_ps(lhs), _mm_load_ps(rhs));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_d_0 = _mm_sub_ps(_mm_loadu_ps(lhs + 0), _mm_loadu_ps(rhs + 0));
|
||||
__m128 xmm_d_1 = _mm_sub_ps(_mm_loadu_ps(lhs + 4), _mm_loadu_ps(rhs + 4));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d_0, xmm_d_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_d_1, xmm_d_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_d = _mm_sub_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs));
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_d, xmm_d, xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
SSD_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __SSE__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,801 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_INT4_STEP_SSE SSD_INT4_SSE
|
||||
#define ACCUM_INT4_STEP_AVX SSD_INT4_AVX
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0xf0f0f0f0);
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0xf0f0f0f0);
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_INT4_GENERAL(m, q, sum) \
|
||||
sum += Int4SquaredDiffTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \
|
||||
Int4SquaredDiffTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)];
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_INT4_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_m), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_q), 4), MASK_INT4_SSE); \
|
||||
xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs)), \
|
||||
4); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_lhs = _mm_and_si128((xmm_m), MASK_INT4_SSE); \
|
||||
xmm_rhs = _mm_and_si128((xmm_q), MASK_INT4_SSE); \
|
||||
xmm_lhs = _mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs), \
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs)), \
|
||||
4); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs, xmm_lhs), ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_INT4_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_m), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_q), 4), MASK_INT4_AVX); \
|
||||
ymm_lhs = \
|
||||
_mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs)), \
|
||||
4); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_lhs = _mm256_and_si256((ymm_m), MASK_INT4_AVX); \
|
||||
ymm_rhs = _mm256_and_si256((ymm_q), MASK_INT4_AVX); \
|
||||
ymm_lhs = \
|
||||
_mm256_srli_epi32(_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs), \
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs)), \
|
||||
4); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs, ymm_lhs), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_lhs), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs_0 = \
|
||||
_mm_and_si128(_mm_slli_epi32((xmm_rhs), 4), MASK_INT4_SSE); \
|
||||
__m128i xmm_lhs_1 = _mm_and_si128((xmm_lhs), MASK_INT4_SSE); \
|
||||
__m128i xmm_rhs_1 = _mm_and_si128((xmm_rhs), MASK_INT4_SSE); \
|
||||
xmm_lhs_0 = \
|
||||
_mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0), \
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0)), \
|
||||
4); \
|
||||
xmm_rhs_0 = \
|
||||
_mm_srli_epi32(_mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1), \
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1)), \
|
||||
4); \
|
||||
xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_lhs_0, xmm_lhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_rhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_rhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_rhs_0), xmm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_lhs), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs_0 = \
|
||||
_mm256_and_si256(_mm256_slli_epi32((ymm_rhs), 4), MASK_INT4_AVX); \
|
||||
__m256i ymm_lhs_1 = _mm256_and_si256((ymm_lhs), MASK_INT4_AVX); \
|
||||
__m256i ymm_rhs_1 = _mm256_and_si256((ymm_rhs), MASK_INT4_AVX); \
|
||||
ymm_lhs_0 = _mm256_srli_epi32( \
|
||||
_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0), \
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0)), \
|
||||
4); \
|
||||
ymm_rhs_0 = _mm256_srli_epi32( \
|
||||
_mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1), \
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1)), \
|
||||
4); \
|
||||
ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_lhs_0, ymm_lhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_rhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_rhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_sum = \
|
||||
_mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_rhs_0), ymm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the square root of value (SSE)
|
||||
#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v))
|
||||
|
||||
//! Compute the square root of value (AVX)
|
||||
#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v))
|
||||
|
||||
//! Compute the square root of value (AVX512)
|
||||
#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v))
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceSSE(const uint8_t *lhs,
|
||||
const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceAVX(const uint8_t *lhs,
|
||||
const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m256i ymm_sum = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = SquaredEuclideanDistanceAVX(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 16, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=32)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 32, 32>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim >> 1));
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim >> 1));
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X1_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_2X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=2)
|
||||
void EuclideanDistanceMatrix<uint8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X2_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_2X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X1_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=2)
|
||||
void EuclideanDistanceMatrix<uint8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X2_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=4)
|
||||
void EuclideanDistanceMatrix<uint8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X4_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=2)
|
||||
void EuclideanDistanceMatrix<uint8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=4)
|
||||
void EuclideanDistanceMatrix<uint8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=8)
|
||||
void EuclideanDistanceMatrix<uint8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=2)
|
||||
void EuclideanDistanceMatrix<uint8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=4)
|
||||
void EuclideanDistanceMatrix<uint8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=8)
|
||||
void EuclideanDistanceMatrix<uint8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=16)
|
||||
void EuclideanDistanceMatrix<uint8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X16_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X16_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=2)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=4)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=8)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=16)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X16_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X16_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=32)
|
||||
void EuclideanDistanceMatrix<uint8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X32_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X32_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,118 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m256i ymm_sum = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
SSD_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __AVX2__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
float SquaredEuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceAVX2(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
float SquaredEuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = SquaredEuclideanDistanceAVX2(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
SquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(m, q, dim, out);
|
||||
*out = std::sqrt(*out);
|
||||
}
|
||||
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,98 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
SSD_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,884 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_INT8_STEP_SSE SSD_INT8_SSE
|
||||
#define ACCUM_INT8_STEP_AVX SSD_INT8_AVX
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate sum of squared difference (GENERAL)
|
||||
#define SSD_INT8_GENERAL(m, q, sum) \
|
||||
{ \
|
||||
int32_t x = m - q; \
|
||||
sum += static_cast<float>(x * x); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (SSE)
|
||||
#define SSD_INT8_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_m), \
|
||||
_mm_sign_epi8(xmm_m, xmm_m)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \
|
||||
_mm_sign_epi8(xmm_q, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_sum = _mm_sub_epi32( \
|
||||
xmm_sum, \
|
||||
_mm_slli_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), \
|
||||
_mm_sign_epi8(xmm_m, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
1)); \
|
||||
}
|
||||
|
||||
//! Calculate sum of squared difference (AVX)
|
||||
#define SSD_INT8_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_m), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_m)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_q, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_sum = _mm256_sub_epi32( \
|
||||
ymm_sum, _mm256_slli_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
1)); \
|
||||
}
|
||||
|
||||
//! Compute the square root of value (SSE)
|
||||
#define SQRT_FP32_SSE(v, ...) _mm_sqrt_ps(_mm_cvtepi32_ps(v))
|
||||
|
||||
//! Compute the square root of value (AVX)
|
||||
#define SQRT_FP32_AVX(v, ...) _mm256_sqrt_ps(_mm256_cvtepi32_ps(v))
|
||||
|
||||
//! Compute the square root of value (AVX512)
|
||||
#define SQRT_FP32_AVX512(v, ...) _mm512_sqrt_ps(_mm512_cvtepi32_ps(v))
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceSSE(const int8_t *lhs,
|
||||
const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum_0 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_1 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0),
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0));
|
||||
xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1),
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1));
|
||||
xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0);
|
||||
xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0),
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0));
|
||||
xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1),
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1));
|
||||
xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0);
|
||||
xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Squared Euclidean Distance
|
||||
static inline float SquaredEuclideanDistanceAVX(const int8_t *lhs,
|
||||
const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
float result = 0.0;
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0),
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0));
|
||||
ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1);
|
||||
|
||||
ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1),
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1));
|
||||
ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs),
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs));
|
||||
ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0),
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0));
|
||||
ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1);
|
||||
|
||||
ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1),
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1));
|
||||
ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs),
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs));
|
||||
ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
}
|
||||
result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_sum);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_sum, xmm_sum));
|
||||
xmm_sum = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs),
|
||||
_mm_madd_epi16(xmm_rhs, xmm_rhs));
|
||||
result += static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = SquaredEuclideanDistanceAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=2)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=4)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=8)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=16)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=32)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim));
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim));
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X1_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_2X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=2)
|
||||
void EuclideanDistanceMatrix<int8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X2_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_2X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X1_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=2)
|
||||
void EuclideanDistanceMatrix<int8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X2_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=4)
|
||||
void EuclideanDistanceMatrix<int8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X4_AVX(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=2)
|
||||
void EuclideanDistanceMatrix<int8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=4)
|
||||
void EuclideanDistanceMatrix<int8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=8)
|
||||
void EuclideanDistanceMatrix<int8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=2)
|
||||
void EuclideanDistanceMatrix<int8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=4)
|
||||
void EuclideanDistanceMatrix<int8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=8)
|
||||
void EuclideanDistanceMatrix<int8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=16)
|
||||
void EuclideanDistanceMatrix<int8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X16_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X16_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X1_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X1_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=2)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X2_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X2_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=4)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X4_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X4_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=8)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X8_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X8_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=16)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X16_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X16_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=32)
|
||||
void EuclideanDistanceMatrix<int8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X32_AVX(m, q, dim, out, SQRT_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X32_SSE(m, q, dim, out, SQRT_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
float result = 0.0;
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0),
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0));
|
||||
ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1);
|
||||
|
||||
ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1),
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1));
|
||||
ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs),
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs));
|
||||
ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_0, ymm_rhs_0),
|
||||
_mm256_min_epi8(ymm_lhs_0, ymm_rhs_0));
|
||||
ymm_lhs_0 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_0 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_0, ymm_lhs_0), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_0, ymm_rhs_0), ymm_sum_1);
|
||||
|
||||
ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs_1, ymm_rhs_1),
|
||||
_mm256_min_epi8(ymm_lhs_1, ymm_rhs_1));
|
||||
ymm_lhs_1 = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs_1 = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs_1, ymm_lhs_1), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs_1, ymm_rhs_1), ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
__m256i ymm_d = _mm256_sub_epi8(_mm256_max_epi8(ymm_lhs, ymm_rhs),
|
||||
_mm256_min_epi8(ymm_lhs, ymm_rhs));
|
||||
ymm_lhs = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(ymm_d));
|
||||
ymm_rhs = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(ymm_d, 1));
|
||||
ymm_sum_0 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_lhs, ymm_lhs), ymm_sum_0);
|
||||
ymm_sum_1 =
|
||||
_mm256_add_epi32(_mm256_madd_epi16(ymm_rhs, ymm_rhs), ymm_sum_1);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
}
|
||||
result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_sum);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_sum, xmm_sum));
|
||||
xmm_sum = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs),
|
||||
_mm_madd_epi16(xmm_rhs, xmm_rhs));
|
||||
result += static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __AVX2__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,59 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
float SquaredEuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceAVX2(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
float SquaredEuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size);
|
||||
float EuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void SquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = SquaredEuclideanDistanceAVX2(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = SquaredEuclideanDistanceSSE(m, q, dim);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void EuclideanDistanceMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
SquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(m, q, dim, out);
|
||||
*out = std::sqrt(*out);
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_euclidean_utility.i"
|
||||
#include "euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Squared Euclidean Distance
|
||||
float SquaredEuclideanDistanceSSE(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum_0 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_1 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0),
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0));
|
||||
xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1),
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1));
|
||||
xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0);
|
||||
xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_0, xmm_rhs_0),
|
||||
_mm_min_epi8(xmm_lhs_0, xmm_rhs_0));
|
||||
xmm_lhs_0 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_0 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs_1, xmm_rhs_1),
|
||||
_mm_min_epi8(xmm_lhs_1, xmm_rhs_1));
|
||||
xmm_lhs_1 = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs_1 = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_0, xmm_lhs_0), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_0, xmm_rhs_0), xmm_sum_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_lhs_1, xmm_lhs_1), xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(xmm_rhs_1, xmm_rhs_1), xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_d = _mm_sub_epi8(_mm_max_epi8(xmm_lhs, xmm_rhs),
|
||||
_mm_min_epi8(xmm_lhs, xmm_rhs));
|
||||
xmm_lhs = _mm_cvtepu8_epi16(xmm_d);
|
||||
xmm_rhs = _mm_cvtepu8_epi16(_mm_unpackhi_epi64(xmm_d, xmm_d));
|
||||
xmm_sum_0 = _mm_add_epi32(_mm_madd_epi16(xmm_lhs, xmm_lhs), xmm_sum_0);
|
||||
xmm_sum_1 = _mm_add_epi32(_mm_madd_epi16(xmm_rhs, xmm_rhs), xmm_sum_1);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
SSD_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
SSD_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
SSD_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
SSD_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
SSD_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
SSD_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
SSD_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
SSD_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
SSD_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
SSD_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
SSD_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
SSD_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
SSD_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
SSD_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
SSD_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -314,248 +314,6 @@ void HammingDistanceMatrix<uint32_t, 1, 1>::Compute(const ValueType *m,
|
||||
*out = static_cast<float>(HammingDistance(m, q, cnt));
|
||||
}
|
||||
|
||||
#if defined(__SSSE3__)
|
||||
//! Compute the distance between matrix and query (UINT32, M=2, N=1)
|
||||
void HammingDistanceMatrix<uint32_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_2X1_AVX(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_2X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=2, N=2)
|
||||
void HammingDistanceMatrix<uint32_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_2X2_AVX(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_2X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=1)
|
||||
void HammingDistanceMatrix<uint32_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X1_AVX(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_4X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=2)
|
||||
void HammingDistanceMatrix<uint32_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X2_AVX(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_4X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=4)
|
||||
void HammingDistanceMatrix<uint32_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X4_AVX(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_4X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=1)
|
||||
void HammingDistanceMatrix<uint32_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_8X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=2)
|
||||
void HammingDistanceMatrix<uint32_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_8X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=4)
|
||||
void HammingDistanceMatrix<uint32_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_8X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=8)
|
||||
void HammingDistanceMatrix<uint32_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_8X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=1)
|
||||
void HammingDistanceMatrix<uint32_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_16X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=2)
|
||||
void HammingDistanceMatrix<uint32_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_16X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=4)
|
||||
void HammingDistanceMatrix<uint32_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_16X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=8)
|
||||
void HammingDistanceMatrix<uint32_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_16X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=16)
|
||||
void HammingDistanceMatrix<uint32_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X16_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_16X16_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=1)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X1_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X1_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=2)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X2_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X2_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=4)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X4_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X4_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=8)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X8_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X8_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=16)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X16_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X16_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=32)
|
||||
void HammingDistanceMatrix<uint32_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X32_AVX(m, q, cnt, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
POPCNT_UINT32_32X32_SSE(m, q, cnt, out, _mm_cvtepi32_ps)
|
||||
#endif
|
||||
}
|
||||
#endif // __SSSE3__
|
||||
|
||||
#if defined(AILEGO_M64)
|
||||
//! Compute the distance between matrix and query (UINT64, M=1, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 1, 1>::Compute(const ValueType *m,
|
||||
@@ -571,167 +329,6 @@ void HammingDistanceMatrix<uint64_t, 1, 1>::Compute(const ValueType *m,
|
||||
*out = static_cast<float>(HammingDistance(m, q, cnt));
|
||||
}
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the distance between matrix and query (UINT64, M=2, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_2X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=2, N=2)
|
||||
void HammingDistanceMatrix<uint64_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_2X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=2)
|
||||
void HammingDistanceMatrix<uint64_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=4)
|
||||
void HammingDistanceMatrix<uint64_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=2)
|
||||
void HammingDistanceMatrix<uint64_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=4)
|
||||
void HammingDistanceMatrix<uint64_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=8)
|
||||
void HammingDistanceMatrix<uint64_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=2)
|
||||
void HammingDistanceMatrix<uint64_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=4)
|
||||
void HammingDistanceMatrix<uint64_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=8)
|
||||
void HammingDistanceMatrix<uint64_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=16)
|
||||
void HammingDistanceMatrix<uint64_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X16_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=1)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X1_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=2)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X2_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=4)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X4_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=8)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X8_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=16)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X16_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=32)
|
||||
void HammingDistanceMatrix<uint64_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X32_AVX(m, q, cnt, out, CONVERT_UINT64_TO_FP32)
|
||||
}
|
||||
#endif // __AVX2__
|
||||
#endif // AILEGO_M64
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=1, N=1)
|
||||
@@ -747,227 +344,6 @@ void HammingSquareRootDistanceMatrix<uint32_t, 1, 1>::Compute(
|
||||
*out = std::sqrt(static_cast<float>(HammingDistance(m, q, cnt)));
|
||||
}
|
||||
|
||||
#if defined(__SSSE3__)
|
||||
//! Compute the distance between matrix and query (UINT32, M=2, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 2, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_2X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#else
|
||||
POPCNT_UINT32_2X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=2, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 2, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_2X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#else
|
||||
POPCNT_UINT32_2X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 4, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#else
|
||||
POPCNT_UINT32_4X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 4, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#else
|
||||
POPCNT_UINT32_4X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=4, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 4, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_4X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#else
|
||||
POPCNT_UINT32_4X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 8, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_8X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 8, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_8X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 8, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_8X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=8, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 8, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_8X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_8X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 16, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_16X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 16, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_16X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 16, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_16X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 16, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_16X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=16, N=16)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 16, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_16X16_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_16X16_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X1_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X1_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X2_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X2_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X4_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X4_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X8_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X8_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=16)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X16_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X16_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT32, M=32, N=32)
|
||||
void HammingSquareRootDistanceMatrix<uint32_t, 32, 32>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 5);
|
||||
#if defined(__AVX2__)
|
||||
POPCNT_UINT32_32X32_AVX(m, q, cnt, out, SQRT_UINT32_TO_FP32_AVX)
|
||||
#else
|
||||
POPCNT_UINT32_32X32_SSE(m, q, cnt, out, SQRT_UINT32_TO_FP32_SSE)
|
||||
#endif
|
||||
}
|
||||
#endif // __SSSE3__
|
||||
|
||||
#if defined(AILEGO_M64)
|
||||
//! Compute the distance between matrix and query (UINT64, M=1, N=1)
|
||||
@@ -983,147 +359,6 @@ void HammingSquareRootDistanceMatrix<uint64_t, 1, 1>::Compute(
|
||||
*out = std::sqrt(static_cast<float>(HammingDistance(m, q, cnt)));
|
||||
}
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the distance between matrix and query (UINT64, M=2, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 2, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_2X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=2, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 2, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_2X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 4, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 4, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=4, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 4, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_4X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 8, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 8, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 8, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=8, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 8, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_8X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 16, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 16, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 16, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 16, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=16, N=16)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 16, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_16X16_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=1)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 1>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X1_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=2)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 2>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X2_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=4)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 4>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X4_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=8)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 8>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X8_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=16)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 16>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X16_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (UINT64, M=32, N=32)
|
||||
void HammingSquareRootDistanceMatrix<uint64_t, 32, 32>::Compute(
|
||||
const ValueType *m, const ValueType *q, size_t dim, float *out) {
|
||||
size_t cnt = (dim >> 6);
|
||||
POPCNT_UINT64_32X32_AVX(m, q, cnt, out, SQRT_UINT64_TO_FP32)
|
||||
}
|
||||
#endif // __AVX2__
|
||||
#endif // AILEGO_M64
|
||||
|
||||
} // namespace ailego
|
||||
|
||||
@@ -83,248 +83,6 @@ struct HammingDistanceMatrix<uint32_t, 1, 1> {
|
||||
float *out);
|
||||
};
|
||||
|
||||
#if defined(__SSSE3__)
|
||||
/*! Hamming Distance Matrix (UINT32, M=2, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 2, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=2, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 2, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=4, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 4, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=4, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 4, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=4, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 4, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=8, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 8, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=8, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 8, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=8, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 8, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=8, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 8, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=16, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 16, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=16, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 16, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=16, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 16, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=16, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 16, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=16, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 16, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT32, M=32, N=32)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint32_t, 32, 32> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
#endif // __SSSE3__
|
||||
|
||||
#if defined(AILEGO_M64)
|
||||
/*! Hamming Distance Matrix (UINT64)
|
||||
*/
|
||||
@@ -381,247 +139,6 @@ struct HammingDistanceMatrix<uint64_t, 1, 1> {
|
||||
float *out);
|
||||
};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
/*! Hamming Distance Matrix (UINT64, M=2, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 2, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=2, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 2, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=4, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 4, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=4, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 4, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=4, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 4, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=8, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 8, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=8, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 8, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=8, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 8, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=8, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 8, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=16, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 16, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=16, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 16, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=16, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 16, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=16, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 16, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=16, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 16, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Distance Matrix (UINT64, M=32, N=32)
|
||||
*/
|
||||
template <>
|
||||
struct HammingDistanceMatrix<uint64_t, 32, 32> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
#endif // __AVX2__
|
||||
#endif // AILEGO_M64
|
||||
|
||||
/*! Hamming Square Root Distance Matrix
|
||||
@@ -656,247 +173,6 @@ struct HammingSquareRootDistanceMatrix<uint32_t, 1, 1> {
|
||||
float *out);
|
||||
};
|
||||
|
||||
#if defined(__SSSE3__)
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=2, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 2, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=2, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 2, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 4, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 4, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=4, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 4, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 8, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 8, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 8, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=8, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 8, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 16, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 16, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 16, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 16, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=16, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 16, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT32, M=32, N=32)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint32_t, 32, 32> {
|
||||
//! Type of value
|
||||
using ValueType = uint32_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
#endif // __SSSE3__
|
||||
|
||||
#if defined(AILEGO_M64)
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=1, N=1)
|
||||
@@ -911,247 +187,6 @@ struct HammingSquareRootDistanceMatrix<uint64_t, 1, 1> {
|
||||
float *out);
|
||||
};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=2, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 2, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=2, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 2, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 4, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 4, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=4, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 4, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 8, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 8, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 8, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=8, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 8, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 16, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 16, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 16, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 16, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=16, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 16, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=1)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 1> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=2)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 2> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=4)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 4> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=8)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 8> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=16)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 16> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
|
||||
/*! Hamming Square Root Distance Matrix (UINT64, M=32, N=32)
|
||||
*/
|
||||
template <>
|
||||
struct HammingSquareRootDistanceMatrix<uint64_t, 32, 32> {
|
||||
//! Type of value
|
||||
using ValueType = uint64_t;
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
static void Compute(const ValueType *m, const ValueType *q, size_t dim,
|
||||
float *out);
|
||||
};
|
||||
#endif // __AVX2__
|
||||
#endif // AILEGO_M64
|
||||
|
||||
} // namespace ailego
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,706 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
// sparse
|
||||
#if defined(__AVX__)
|
||||
const static __m128i SHUFFLE_MASK256[256] = {
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, -127, -127),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 5,
|
||||
4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7,
|
||||
6, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7,
|
||||
6, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 7,
|
||||
6, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 9,
|
||||
8, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 11,
|
||||
10, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 13,
|
||||
12, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11, 10,
|
||||
9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 15, 14),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11, 10,
|
||||
9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6,
|
||||
5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6,
|
||||
5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127, 15,
|
||||
14, 13, 12, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7,
|
||||
6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7,
|
||||
6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7,
|
||||
6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9,
|
||||
8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9,
|
||||
8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9,
|
||||
8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9,
|
||||
8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5,
|
||||
4),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2),
|
||||
_mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
};
|
||||
|
||||
constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536;
|
||||
|
||||
float InnerProductSparseInSegmentAVX(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const Float16 *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const Float16 *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
// handle if the first dim is zero
|
||||
bool m_zero = false;
|
||||
Float16 m_zero_value{0.0f};
|
||||
if (m_sparse_count > 0 && m_sparse_index[0] == 0) {
|
||||
m_sparse_count--;
|
||||
m_sparse_index++;
|
||||
m_zero_value = *m_sparse_value++;
|
||||
m_zero = true;
|
||||
}
|
||||
|
||||
bool q_zero = false;
|
||||
Float16 q_zero_value{0.0f};
|
||||
if (q_sparse_count > 0 && q_sparse_index[0] == 0) {
|
||||
q_sparse_count--;
|
||||
q_sparse_index++;
|
||||
q_zero_value = *q_sparse_value++;
|
||||
q_zero = true;
|
||||
}
|
||||
|
||||
if (m_zero && q_zero) {
|
||||
sum = m_zero_value * q_zero_value;
|
||||
}
|
||||
|
||||
size_t i1 = 0, i2 = 0;
|
||||
size_t end1 = m_sparse_count / 8 * 8;
|
||||
size_t end2 = q_sparse_count / 8 * 8;
|
||||
|
||||
uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH];
|
||||
uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH];
|
||||
|
||||
Float16 *val_start_1 = reinterpret_cast<Float16 *>(fixed_buffer_1);
|
||||
Float16 *val_start_2 = reinterpret_cast<Float16 *>(fixed_buffer_2);
|
||||
|
||||
Float16 *val_1 = val_start_1;
|
||||
Float16 *val_2 = val_start_2;
|
||||
|
||||
if (i1 < end1 && i2 < end2) {
|
||||
while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
}
|
||||
|
||||
while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
}
|
||||
|
||||
__m128i mm_index_m =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
__m128i mm_index_q =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
|
||||
while (true) {
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "index 1: " << std::endl;
|
||||
print_data16(&mm_index_m);
|
||||
|
||||
std::cout << "index 2: " << std::endl;
|
||||
print_data16(&mm_index_q);
|
||||
#endif
|
||||
|
||||
__m128i mm_cmp_res =
|
||||
_mm_cmpistrm(mm_index_q, mm_index_m,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "cmp res: " << std::endl;
|
||||
print_data16(&mm_cmp_res);
|
||||
#endif
|
||||
|
||||
int r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
if (r) {
|
||||
int r1 = r;
|
||||
|
||||
__m128i v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1]));
|
||||
__m128i vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]);
|
||||
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i *>(val_1), vs);
|
||||
val_1 += _mm_popcnt_u32(r1);
|
||||
|
||||
mm_cmp_res = _mm_cmpistrm(
|
||||
mm_index_m, mm_index_q,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
r1 = r;
|
||||
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2]));
|
||||
vs = _mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]);
|
||||
|
||||
_mm_storeu_si128(reinterpret_cast<__m128i *>(val_2), vs);
|
||||
val_2 += _mm_popcnt_u32(r1);
|
||||
}
|
||||
|
||||
const uint16_t id1_max = m_sparse_index[i1 + 7];
|
||||
|
||||
if (id1_max <= q_sparse_index[i2 + 7]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
mm_index_m = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
}
|
||||
|
||||
if (id1_max >= q_sparse_index[i2 + 7]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
mm_index_q = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_scalar:
|
||||
while (i1 < m_sparse_count && i2 < q_sparse_count) {
|
||||
if (m_sparse_index[i1] == q_sparse_index[i2]) {
|
||||
*val_1++ = m_sparse_value[i1];
|
||||
*val_2++ = q_sparse_value[i2];
|
||||
|
||||
++i1;
|
||||
++i2;
|
||||
} else if (m_sparse_index[i1] < q_sparse_index[i2]) {
|
||||
++i1;
|
||||
} else {
|
||||
++i2;
|
||||
}
|
||||
}
|
||||
|
||||
size_t res_num = val_1 - val_start_1;
|
||||
|
||||
size_t res_num8 = res_num / 8 * 8;
|
||||
|
||||
if (res_num8) {
|
||||
__m256 sum256 = _mm256_setzero_ps();
|
||||
|
||||
for (size_t k = 0; k < res_num8; k += 8) {
|
||||
__m256 ymm_1 =
|
||||
_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_1 + k)));
|
||||
__m256 ymm_2 =
|
||||
_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(val_start_2 + k)));
|
||||
ACCUM_FP32_STEP_AVX(ymm_1, ymm_2, sum256);
|
||||
}
|
||||
|
||||
sum += HorizontalAdd_FP32_V256(sum256);
|
||||
}
|
||||
|
||||
for (size_t k = res_num8; k < res_num; ++k)
|
||||
sum += val_start_1[k] * val_start_2[k];
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
#endif // __AVX__
|
||||
|
||||
|
||||
#if defined(__AVX__)
|
||||
void InnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out) {
|
||||
ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, )
|
||||
}
|
||||
|
||||
void MinusInnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out) {
|
||||
ACCUM_FP16_1X1_AVX(lhs, rhs, size, out, 0ull, NEGATE_FP32_GENERAL)
|
||||
}
|
||||
#endif
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,766 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
//! Inner Product
|
||||
float InnerProductAVX512FP16(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
__m512h zmm_sum_0 = _mm512_setzero_ph();
|
||||
__m512h zmm_sum_1 = _mm512_setzero_ph();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 0), _mm512_load_ph(rhs + 0),
|
||||
zmm_sum_0)
|
||||
|
||||
FMA_FP16_AVX512FP16(_mm512_load_ph(lhs + 32), _mm512_load_ph(rhs + 32),
|
||||
zmm_sum_1)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
FMA_FP16_AVX512FP16(_mm512_load_ph(lhs), _mm512_load_ph(rhs), zmm_sum_0)
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 0), _mm512_loadu_ph(rhs + 0),
|
||||
zmm_sum_0)
|
||||
|
||||
FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs + 32), _mm512_loadu_ph(rhs + 32),
|
||||
zmm_sum_1)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
FMA_FP16_AVX512FP16(_mm512_loadu_ph(lhs), _mm512_loadu_ph(rhs), zmm_sum_0)
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ph(zmm_sum_0, zmm_sum_1);
|
||||
|
||||
if (lhs != last) {
|
||||
__mmask32 mask = (__mmask32)((1 << (last - lhs)) - 1);
|
||||
__m512i zmm_undefined = _mm512_undefined_epi32();
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ph(
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, lhs)),
|
||||
_mm512_castsi512_ph(_mm512_mask_loadu_epi16(zmm_undefined, mask, rhs)),
|
||||
zmm_sum_0, mask);
|
||||
}
|
||||
|
||||
return HorizontalAdd_FP16_V512(zmm_sum_0);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// sparse
|
||||
#if defined(__AVX512FP16__)
|
||||
constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536;
|
||||
|
||||
float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const Float16 *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const Float16 *q_sparse_value) {
|
||||
const static __m128i SHUFFLE_MASK256[256] = {
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, -127, -127),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
7, 6, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
7, 6, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
7, 6, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
9, 8, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 9, 8, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 9, 8, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
11, 10, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 13, 12, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
13, 12, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 13, 12, 11,
|
||||
10, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7,
|
||||
6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7,
|
||||
6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7,
|
||||
6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, 15, 14),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 5, 4,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 7, 6,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 7, 6, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 9, 8, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 11,
|
||||
10, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7,
|
||||
6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7,
|
||||
6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7,
|
||||
6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5,
|
||||
4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5,
|
||||
4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7,
|
||||
6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7,
|
||||
6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7,
|
||||
6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3,
|
||||
2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
15, 14, 13, 12, 11, 10),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 11, 10, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 11, 10, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 11, 10, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
5, 4, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 5, 4, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 11, 10, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
7, 6, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
7, 6, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13,
|
||||
12, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
9, 8, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
9, 8, 3, 2),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
9, 8, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, 15, 14, 13, 12, 11, 10,
|
||||
9, 8, 7, 6),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
3, 2),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6,
|
||||
5, 4),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 1, 0),
|
||||
_mm_set_epi8(-127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2),
|
||||
_mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
};
|
||||
|
||||
float sum = 0.0f;
|
||||
|
||||
// handle if the first dim is zero
|
||||
bool m_zero = false;
|
||||
Float16 m_zero_value{0.0f};
|
||||
if (m_sparse_count > 0 && m_sparse_index[0] == 0) {
|
||||
m_sparse_count--;
|
||||
m_sparse_index++;
|
||||
m_zero_value = *m_sparse_value++;
|
||||
m_zero = true;
|
||||
}
|
||||
|
||||
bool q_zero = false;
|
||||
Float16 q_zero_value{0.0f};
|
||||
if (q_sparse_count > 0 && q_sparse_index[0] == 0) {
|
||||
q_sparse_count--;
|
||||
q_sparse_index++;
|
||||
q_zero_value = *q_sparse_value++;
|
||||
q_zero = true;
|
||||
}
|
||||
|
||||
if (m_zero && q_zero) {
|
||||
sum = m_zero_value * q_zero_value;
|
||||
}
|
||||
|
||||
size_t i1 = 0, i2 = 0;
|
||||
size_t end1 = m_sparse_count / 8 * 8;
|
||||
size_t end2 = q_sparse_count / 8 * 8;
|
||||
|
||||
uint16_t fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH];
|
||||
uint16_t fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH];
|
||||
|
||||
Float16 *val_start_1 = reinterpret_cast<Float16 *>(fixed_buffer_1);
|
||||
Float16 *val_start_2 = reinterpret_cast<Float16 *>(fixed_buffer_2);
|
||||
|
||||
Float16 *val_1 = val_start_1;
|
||||
Float16 *val_2 = val_start_2;
|
||||
|
||||
if (i1 < end1 && i2 < end2) {
|
||||
while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
}
|
||||
|
||||
while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
}
|
||||
|
||||
__m128i mm_index_m =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
__m128i mm_index_q =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
|
||||
while (true) {
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "index 1: " << std::endl;
|
||||
print_data16(&mm_index_m);
|
||||
|
||||
std::cout << "index 2: " << std::endl;
|
||||
print_data16(&mm_index_q);
|
||||
#endif
|
||||
|
||||
__m128i mm_cmp_res =
|
||||
_mm_cmpistrm(mm_index_q, mm_index_m,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "cmp res: " << std::endl;
|
||||
print_data16(&mm_cmp_res);
|
||||
#endif
|
||||
|
||||
int r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
if (r) {
|
||||
int r1 = r;
|
||||
|
||||
__m128i v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1]));
|
||||
__m128h vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]));
|
||||
|
||||
_mm_storeu_ph(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r1);
|
||||
|
||||
mm_cmp_res = _mm_cmpistrm(
|
||||
mm_index_m, mm_index_q,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
r1 = r;
|
||||
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2]));
|
||||
vs = _mm_castsi128_ph(_mm_shuffle_epi8(v, SHUFFLE_MASK256[r1]));
|
||||
|
||||
_mm_storeu_ph(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r1);
|
||||
}
|
||||
|
||||
const uint16_t id1_max = m_sparse_index[i1 + 7];
|
||||
|
||||
if (id1_max <= q_sparse_index[i2 + 7]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
mm_index_m = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
}
|
||||
|
||||
if (id1_max >= q_sparse_index[i2 + 7]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
mm_index_q = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_scalar:
|
||||
while (i1 < m_sparse_count && i2 < q_sparse_count) {
|
||||
if (m_sparse_index[i1] == q_sparse_index[i2]) {
|
||||
*val_1++ = m_sparse_value[i1];
|
||||
*val_2++ = q_sparse_value[i2];
|
||||
|
||||
++i1;
|
||||
++i2;
|
||||
} else if (m_sparse_index[i1] < q_sparse_index[i2]) {
|
||||
++i1;
|
||||
} else {
|
||||
++i2;
|
||||
}
|
||||
}
|
||||
|
||||
size_t res_num = val_1 - val_start_1;
|
||||
|
||||
size_t res_num8 = res_num / 8 * 8;
|
||||
|
||||
if (res_num8) {
|
||||
__m128h sum128 = _mm_set1_ph(0);
|
||||
|
||||
for (size_t k = 0; k < res_num8; k += 8) {
|
||||
sum128 = _mm_add_ph(sum128, _mm_mul_ph(_mm_loadu_ph(val_start_1 + k),
|
||||
_mm_loadu_ph(val_start_2 + k)));
|
||||
}
|
||||
|
||||
Float16 __attribute__((aligned(16))) tmp_res[8];
|
||||
_mm_store_ph(tmp_res, sum128);
|
||||
sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3] + tmp_res[4] +
|
||||
tmp_res[5] + tmp_res[6] + tmp_res[7]);
|
||||
}
|
||||
|
||||
for (size_t k = res_num8; k < res_num; ++k)
|
||||
sum += val_start_1[k] * val_start_2[k];
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
#endif // __AVX512FP16__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void InnerProductAVX512(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out) {
|
||||
ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, )
|
||||
}
|
||||
|
||||
void MinusInnerProductAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out) {
|
||||
ACCUM_FP16_1X1_AVX512(lhs, rhs, size, out, 0ull, NEGATE_FP32_GENERAL)
|
||||
}
|
||||
#endif //__AVX512F__
|
||||
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
float InnerProductNEON(const Float16 *lhs, const Float16 *rhs, size_t size);
|
||||
float MinusInnerProductNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
void InnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out);
|
||||
void MinusInnerProductAVX(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out);
|
||||
float InnerProductSparseInSegmentAVX(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const Float16 *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const Float16 *q_sparse_value);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void InnerProductAVX512(const Float16 *lhs, const Float16 *rhs, size_t size,
|
||||
float *out);
|
||||
void MinusInnerProductAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *out);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
float InnerProductAVX512FP16(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size);
|
||||
float InnerProductSparseInSegmentAVX512FP16(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const Float16 *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const Float16 *q_sparse_value);
|
||||
#endif
|
||||
|
||||
#if (defined(__F16C__) && defined(__AVX__)) || \
|
||||
(defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void InnerProductMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = InnerProductNEON(m, q, dim);
|
||||
#else
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
*out = InnerProductAVX512FP16(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif //__AVX512FP16__
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
InnerProductAVX512(m, q, dim, out);
|
||||
return;
|
||||
}
|
||||
#endif //__AVX512F__
|
||||
InnerProductAVX(m, q, dim, out);
|
||||
#endif //__ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP16, M=1, N=1)
|
||||
void MinusInnerProductMatrix<Float16, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = MinusInnerProductNEON(m, q, dim);
|
||||
#else
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
*out = -InnerProductAVX512FP16(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif //__AVX512FP16__
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
MinusInnerProductAVX512(m, q, dim, out);
|
||||
return;
|
||||
}
|
||||
#endif //__AVX512F__
|
||||
|
||||
MinusInnerProductAVX(m, q, dim, out);
|
||||
|
||||
#endif //__ARM_NEON
|
||||
}
|
||||
|
||||
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)
|
||||
|
||||
// sparse
|
||||
float InnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const Float16 *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const Float16 *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
size_t m_i = 0;
|
||||
size_t q_i = 0;
|
||||
while (m_i < m_sparse_count && q_i < q_sparse_count) {
|
||||
if (m_sparse_index[m_i] == q_sparse_index[q_i]) {
|
||||
sum += m_sparse_value[m_i] * q_sparse_value[q_i];
|
||||
|
||||
++m_i;
|
||||
++q_i;
|
||||
} else if (m_sparse_index[m_i] < q_sparse_index[q_i]) {
|
||||
++m_i;
|
||||
} else {
|
||||
++q_i;
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
template <>
|
||||
float MinusInnerProductSparseMatrix<Float16>::
|
||||
ComputeInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const ValueType *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const ValueType *q_sparse_value) {
|
||||
#if defined(__AVX512FP16__)
|
||||
return InnerProductSparseInSegmentAVX512FP16(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#elif defined(__AVX__)
|
||||
return InnerProductSparseInSegmentAVX(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
|
||||
#else
|
||||
return InnerProductSparseInSegment(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
float InnerProductNEON(const Float16 *lhs, const Float16 *rhs, size_t size) {
|
||||
float score;
|
||||
|
||||
ACCUM_FP16_1X1_NEON(lhs, rhs, size, &score, 0ull, )
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
float MinusInnerProductNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size) {
|
||||
float score;
|
||||
|
||||
ACCUM_FP16_1X1_NEON(lhs, rhs, size, &score, 0ull, NEGATE_FP32_GENERAL)
|
||||
|
||||
return score;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,94 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__)
|
||||
//! Inner Product
|
||||
float InnerProductAVX(const float *lhs, const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
ymm_sum_0 =
|
||||
_mm256_fmadd_ps(_mm256_load_ps(lhs), _mm256_load_ps(rhs), ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
ymm_sum_0 = _mm256_fmadd_ps(_mm256_loadu_ps(lhs), _mm256_loadu_ps(rhs),
|
||||
ymm_sum_0);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP32_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP32_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP32_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP32_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __AVX__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Inner Product
|
||||
float InnerProductAVX512(const float *lhs, const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
FMA_FP32_AVX512(_mm512_load_ps(lhs + 0), _mm512_load_ps(rhs + 0),
|
||||
zmm_sum_0)
|
||||
|
||||
FMA_FP32_AVX512(_mm512_load_ps(lhs + 16), _mm512_load_ps(rhs + 16),
|
||||
zmm_sum_1)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
FMA_FP32_AVX512(_mm512_load_ps(lhs), _mm512_load_ps(rhs), zmm_sum_0)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 0), _mm512_loadu_ps(rhs + 0),
|
||||
zmm_sum_0)
|
||||
|
||||
FMA_FP32_AVX512(_mm512_loadu_ps(lhs + 16), _mm512_loadu_ps(rhs + 16),
|
||||
zmm_sum_1)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
FMA_FP32_AVX512(_mm512_loadu_ps(lhs), _mm512_loadu_ps(rhs), zmm_sum_0)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1);
|
||||
__m512 zmm_undefined = _mm512_undefined_ps();
|
||||
zmm_sum_0 = _mm512_mask3_fmadd_ps(
|
||||
_mm512_mask_loadu_ps(zmm_undefined, mask, lhs),
|
||||
_mm512_mask_loadu_ps(zmm_undefined, mask, rhs), zmm_sum_0, mask);
|
||||
}
|
||||
return HorizontalAdd_FP32_V512(zmm_sum_0);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,97 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
float InnerProductNEON(const float *lhs, const float *rhs, size_t size);
|
||||
float MinusInnerProductNEON(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
float InnerProductAVX512(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
float InnerProductAVX(const float *lhs, const float *rhs, size_t size);
|
||||
float MinusInnerProductAVX(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__)
|
||||
float InnerProductSSE(const float *lhs, const float *rhs, size_t size);
|
||||
float MinusInnerProductSSE(const float *lhs, const float *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__) || defined(__ARM_NEON)
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void InnerProductMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = InnerProductNEON(m, q, dim);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
if (dim > 15) {
|
||||
*out = InnerProductAVX512(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
if (dim > 7) {
|
||||
*out = InnerProductAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX__
|
||||
*out = InnerProductSSE(m, q, dim);
|
||||
#endif // __ARM_NEON
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (FP32, M=1, N=1)
|
||||
void MinusInnerProductMatrix<float, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
*out = -InnerProductNEON(m, q, dim);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
if (dim > 15) {
|
||||
*out = -InnerProductAVX512(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
if (dim > 7) {
|
||||
*out = -InnerProductAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif // __AVX__
|
||||
*out = -InnerProductSSE(m, q, dim);
|
||||
#endif // __ARM_NEON
|
||||
}
|
||||
|
||||
#endif
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,57 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Inner Product
|
||||
float InnerProductNEON(const float *lhs, const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs + 0), vld1q_f32(rhs + 0));
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, vld1q_f32(lhs + 4), vld1q_f32(rhs + 4));
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, vld1q_f32(lhs), vld1q_f32(rhs));
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __ARM_NEON
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,351 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Inner Product
|
||||
float InnerProductSSE(const float *lhs, const float *rhs, size_t size) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
__m128 xmm_sum_0 = _mm_setzero_ps();
|
||||
__m128 xmm_sum_1 = _mm_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_load_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_load_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_load_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_load_ps(rhs + 4);
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
xmm_sum_0 = _mm_fmadd_ps(_mm_load_ps(lhs), _mm_load_ps(rhs), xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4);
|
||||
xmm_sum_0 = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum_0);
|
||||
xmm_sum_1 = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
xmm_sum_0 = _mm_fmadd_ps(_mm_loadu_ps(lhs), _mm_loadu_ps(rhs), xmm_sum_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V128(_mm_add_ps(xmm_sum_0, xmm_sum_1));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __SSE__
|
||||
|
||||
// #if 1
|
||||
#if defined(__SSE4_1__)
|
||||
const static __m128i SHUFFLE_MASK16[16] = {
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, -127, -127),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5,
|
||||
4),
|
||||
_mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
};
|
||||
|
||||
constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536;
|
||||
|
||||
float InnerProductSparseInSegmentSSE(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
// handle if the first dim is zero
|
||||
bool m_zero = false;
|
||||
float m_zero_value = 0.0f;
|
||||
if (m_sparse_count > 0 && m_sparse_index[0] == 0) {
|
||||
m_sparse_count--;
|
||||
m_sparse_index++;
|
||||
m_zero_value = *m_sparse_value++;
|
||||
m_zero = true;
|
||||
}
|
||||
|
||||
bool q_zero = false;
|
||||
float q_zero_value = 0.0f;
|
||||
if (q_sparse_count > 0 && q_sparse_index[0] == 0) {
|
||||
q_sparse_count--;
|
||||
q_sparse_index++;
|
||||
q_zero_value = *q_sparse_value++;
|
||||
q_zero = true;
|
||||
}
|
||||
|
||||
if (m_zero && q_zero) {
|
||||
sum = m_zero_value * q_zero_value;
|
||||
}
|
||||
|
||||
size_t i1 = 0, i2 = 0;
|
||||
size_t end1 = m_sparse_count / 8 * 8;
|
||||
size_t end2 = q_sparse_count / 8 * 8;
|
||||
|
||||
// std::vector<float> mem1;
|
||||
// std::vector<float> mem2;
|
||||
|
||||
float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH];
|
||||
float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH];
|
||||
|
||||
float *val_start_1 = fixed_buffer_1;
|
||||
float *val_start_2 = fixed_buffer_2;
|
||||
|
||||
// uint32_t max_count = std::max(m_sparse_count, q_sparse_count);
|
||||
|
||||
// if (MAX_SPARSE_BUFFER_LENGTH < max_count) {
|
||||
// mem1.reserve(max_count);
|
||||
// mem2.reserve(max_count);
|
||||
|
||||
// val_start_1 = mem1.data();
|
||||
// val_start_2 = mem2.data();
|
||||
// }
|
||||
|
||||
float *val_1 = val_start_1;
|
||||
float *val_2 = val_start_2;
|
||||
|
||||
if (i1 < end1 && i2 < end2) {
|
||||
while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
}
|
||||
|
||||
while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
}
|
||||
|
||||
__m128i mm_index_m =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
__m128i mm_index_q =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
|
||||
while (true) {
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "index 1: " << std::endl;
|
||||
print_data16(&mm_index_m);
|
||||
|
||||
std::cout << "index 2: " << std::endl;
|
||||
print_data16(&mm_index_q);
|
||||
#endif
|
||||
|
||||
__m128i mm_cmp_res =
|
||||
_mm_cmpistrm(mm_index_q, mm_index_m,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "cmp res: " << std::endl;
|
||||
print_data16(&mm_cmp_res);
|
||||
#endif
|
||||
|
||||
int r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
if (r) {
|
||||
int r1 = r & 15;
|
||||
|
||||
__m128i v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1]));
|
||||
__m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r1);
|
||||
|
||||
int r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r2);
|
||||
|
||||
mm_cmp_res = _mm_cmpistrm(
|
||||
mm_index_m, mm_index_q,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
r1 = r & 15;
|
||||
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r1);
|
||||
|
||||
r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r2);
|
||||
}
|
||||
|
||||
const uint16_t id1_max = m_sparse_index[i1 + 7];
|
||||
|
||||
if (id1_max <= q_sparse_index[i2 + 7]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
mm_index_m = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
}
|
||||
|
||||
if (id1_max >= q_sparse_index[i2 + 7]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
mm_index_q = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_scalar:
|
||||
while (i1 < m_sparse_count && i2 < q_sparse_count) {
|
||||
if (m_sparse_index[i1] == q_sparse_index[i2]) {
|
||||
*val_1++ = m_sparse_value[i1];
|
||||
*val_2++ = q_sparse_value[i2];
|
||||
|
||||
++i1;
|
||||
++i2;
|
||||
} else if (m_sparse_index[i1] < q_sparse_index[i2]) {
|
||||
++i1;
|
||||
} else {
|
||||
++i2;
|
||||
}
|
||||
}
|
||||
|
||||
size_t res_num = val_1 - val_start_1;
|
||||
|
||||
// if (res_num != val_2 - val_start_2) {
|
||||
// std::cerr << "size mismatch!" << std::endl;
|
||||
// }
|
||||
|
||||
size_t res_num4 = res_num / 4 * 4;
|
||||
|
||||
if (res_num4) {
|
||||
__m128 sum128 = _mm_set1_ps(0);
|
||||
|
||||
for (size_t k = 0; k < res_num4; k += 4) {
|
||||
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k),
|
||||
_mm_loadu_ps(val_start_2 + k)));
|
||||
}
|
||||
|
||||
float __attribute__((aligned(16))) tmp_res[4];
|
||||
_mm_store_ps(tmp_res, sum128);
|
||||
sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]);
|
||||
}
|
||||
|
||||
for (size_t k = res_num4; k < res_num; ++k)
|
||||
sum += val_start_1[k] * val_start_2[k];
|
||||
|
||||
return sum;
|
||||
}
|
||||
#else
|
||||
float InnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
size_t m_i = 0;
|
||||
size_t q_i = 0;
|
||||
while (m_i < m_sparse_count && q_i < q_sparse_count) {
|
||||
if (m_sparse_index[m_i] == q_sparse_index[q_i]) {
|
||||
sum += m_sparse_value[m_i] * q_sparse_value[q_i];
|
||||
|
||||
++m_i;
|
||||
++q_i;
|
||||
} else if (m_sparse_index[m_i] < q_sparse_index[q_i]) {
|
||||
++m_i;
|
||||
} else {
|
||||
++q_i;
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
template <>
|
||||
float MinusInnerProductSparseMatrix<float>::ComputeInnerProductSparseInSegment(
|
||||
uint32_t m_sparse_count, const uint16_t *m_sparse_index,
|
||||
const ValueType *m_sparse_value, uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index, const ValueType *q_sparse_value) {
|
||||
#if defined(__SSE4_1__)
|
||||
return InnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#else
|
||||
return InnerProductSparseInSegment(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,803 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_INT4_STEP_SSE FMA_INT4_SSE
|
||||
#define ACCUM_INT4_STEP_AVX FMA_INT4_AVX
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Four-bits Convert Table
|
||||
static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1};
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f);
|
||||
static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f);
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
static const __m128i INT4_LOOKUP_SSE =
|
||||
_mm_load_si128((const __m128i *)Int4ConvertTable);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f);
|
||||
static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f);
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
static const __m256i INT4_LOOKUP_AVX =
|
||||
_mm256_load_si256((const __m256i *)Int4ConvertTable);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT4_GENERAL(m, q, sum) \
|
||||
sum += Int4MulTable[(((m) << 4) & 0xf0) | (((q) >> 0) & 0xf)] + \
|
||||
Int4MulTable[(((m) >> 0) & 0xf0) | (((q) >> 4) & 0xf)];
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT4_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128((xmm_m), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs = _mm_shuffle_epi8(INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128((xmm_q), MASK_INT4_SSE)); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
xmm_lhs = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_m), 4), MASK_INT4_SSE)); \
|
||||
xmm_rhs = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_q), 4), MASK_INT4_SSE)); \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT4_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_m), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_q), MASK_INT4_AVX)); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
ymm_lhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_m), 4), MASK_INT4_AVX)); \
|
||||
ymm_rhs = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_q), 4), MASK_INT4_AVX)); \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_lhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0); \
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1); \
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0); \
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1); \
|
||||
xmm_lhs_0 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_lhs_1 = _mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1), \
|
||||
ONES_INT16_SSE); \
|
||||
xmm_sum = _mm_add_epi32(_mm_add_epi32(xmm_lhs_0, xmm_lhs_1), xmm_sum); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query
|
||||
#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_lhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0); \
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1); \
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0); \
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1); \
|
||||
ymm_lhs_0 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_lhs_1 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1), \
|
||||
ONES_INT16_AVX); \
|
||||
ymm_sum = \
|
||||
_mm256_add_epi32(_mm256_add_epi32(ymm_lhs_0, ymm_lhs_1), ymm_sum); \
|
||||
}
|
||||
|
||||
//! Reverse sign of value (SSE)
|
||||
#define NEGATE_FP32_SSE(v, ...) \
|
||||
_mm_xor_ps(_mm_cvtepi32_ps(v), NEGZEROS_FP32_SSE)
|
||||
|
||||
//! Reverse sign of value (AVX)
|
||||
#define NEGATE_FP32_AVX(v, ...) \
|
||||
_mm256_xor_ps(_mm256_cvtepi32_ps(v), NEGZEROS_FP32_AVX)
|
||||
|
||||
//! Reverse sign of value (AVX512)
|
||||
#define NEGATE_FP32_AVX512(v, ...) \
|
||||
_mm512_xor_ps(_mm512_cvtepi32_ps(v), NEGZEROS_FP32_AVX512)
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Inner Product
|
||||
static inline float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Inner Product
|
||||
static inline float InnerProductAVX(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
__m256i ymm_sum = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void InnerProductMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = InnerProductAVX(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = InnerProductSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=1)
|
||||
void InnerProductMatrix<uint8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=2)
|
||||
void InnerProductMatrix<uint8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=1)
|
||||
void InnerProductMatrix<uint8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=2)
|
||||
void InnerProductMatrix<uint8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=4)
|
||||
void InnerProductMatrix<uint8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=1)
|
||||
void InnerProductMatrix<uint8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=2)
|
||||
void InnerProductMatrix<uint8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=4)
|
||||
void InnerProductMatrix<uint8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=8)
|
||||
void InnerProductMatrix<uint8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=1)
|
||||
void InnerProductMatrix<uint8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=2)
|
||||
void InnerProductMatrix<uint8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=4)
|
||||
void InnerProductMatrix<uint8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=8)
|
||||
void InnerProductMatrix<uint8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=16)
|
||||
void InnerProductMatrix<uint8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=1)
|
||||
void InnerProductMatrix<uint8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=2)
|
||||
void InnerProductMatrix<uint8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=4)
|
||||
void InnerProductMatrix<uint8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=8)
|
||||
void InnerProductMatrix<uint8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=16)
|
||||
void InnerProductMatrix<uint8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=32)
|
||||
void InnerProductMatrix<uint8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT4_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = -InnerProductAVX(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = -InnerProductSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_2X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=2, N=2)
|
||||
void MinusInnerProductMatrix<uint8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_2X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=2)
|
||||
void MinusInnerProductMatrix<uint8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=4, N=4)
|
||||
void MinusInnerProductMatrix<uint8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT4_4X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=2)
|
||||
void MinusInnerProductMatrix<uint8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=4)
|
||||
void MinusInnerProductMatrix<uint8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=8, N=8)
|
||||
void MinusInnerProductMatrix<uint8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_8X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=2)
|
||||
void MinusInnerProductMatrix<uint8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=4)
|
||||
void MinusInnerProductMatrix<uint8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=8)
|
||||
void MinusInnerProductMatrix<uint8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=16, N=16)
|
||||
void MinusInnerProductMatrix<uint8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_16X16_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=2)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=4)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=8)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=16)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X16_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=32, N=32)
|
||||
void MinusInnerProductMatrix<uint8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT4_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT4_32X32_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,123 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Inner Product
|
||||
float InnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
__m256i ymm_sum = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum)
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
ymm_sum = _mm256_add_epi32(_mm256_set_m128i(_mm_setzero_si128(), xmm_sum),
|
||||
ymm_sum);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
float MinusInnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
return -InnerProductAVX2(lhs, rhs, size);
|
||||
}
|
||||
|
||||
#endif // __AVX2__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
float InnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs, size_t size);
|
||||
float MinusInnerProductAVX2(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size);
|
||||
float MinusInnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void InnerProductMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = InnerProductAVX2(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = InnerProductSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT4, M=1, N=1)
|
||||
void MinusInnerProductMatrix<uint8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 63) {
|
||||
*out = MinusInnerProductAVX2(m, q, dim >> 1);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = MinusInnerProductSSE(m, q, dim >> 1);
|
||||
}
|
||||
|
||||
#endif //__SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,101 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int4.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Inner Product
|
||||
float InnerProductSSE(const uint8_t *lhs, const uint8_t *rhs, size_t size) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
float MinusInnerProductSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size) {
|
||||
return -InnerProductSSE(lhs, rhs, size);
|
||||
}
|
||||
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,841 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#define ACCUM_INT8_STEP_SSE FMA_INT8_SSE
|
||||
#define ACCUM_INT8_STEP_AVX FMA_INT8_AVX
|
||||
|
||||
#if defined(__AVX512F__) && !defined(__AVX512DQ__)
|
||||
#define _mm512_xor_ps(a, b) \
|
||||
_mm512_castsi512_ps( \
|
||||
_mm512_xor_epi32(_mm512_castps_si512(a), _mm512_castps_si512(b)))
|
||||
#endif // __AVX512DQ__
|
||||
|
||||
#if defined(__SSE__)
|
||||
static const __m128 NEGZEROS_FP32_SSE = _mm_set1_ps(-0.0f);
|
||||
#endif // __SSE__
|
||||
|
||||
#if defined(__AVX__)
|
||||
static const __m256 NEGZEROS_FP32_AVX = _mm256_set1_ps(-0.0f);
|
||||
#endif // __AVX__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
static const __m512 NEGZEROS_FP32_AVX512 = _mm512_set1_ps(-0.0f);
|
||||
#endif // __AVX512F__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Reverse sign of value (SSE)
|
||||
#define NEGATE_FP32_SSE(v, ...) \
|
||||
_mm_xor_ps(_mm_cvtepi32_ps(v), NEGZEROS_FP32_SSE)
|
||||
|
||||
//! Reverse sign of value (AVX)
|
||||
#define NEGATE_FP32_AVX(v, ...) \
|
||||
_mm256_xor_ps(_mm256_cvtepi32_ps(v), NEGZEROS_FP32_AVX)
|
||||
|
||||
//! Reverse sign of value (AVX512)
|
||||
#define NEGATE_FP32_AVX512(v, ...) \
|
||||
_mm512_xor_ps(_mm512_cvtepi32_ps(v), NEGZEROS_FP32_AVX512)
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT8_GENERAL(m, q, sum) sum += static_cast<float>(m * q);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT8_SSE(xmm_m, xmm_q, xmm_sum) \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16( \
|
||||
_mm_maddubs_epi16(_mm_abs_epi8(xmm_q), _mm_sign_epi8(xmm_m, xmm_q)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum);
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT8_AVX(ymm_m, ymm_q, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_q), \
|
||||
_mm256_sign_epi8(ymm_m, ymm_q)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum);
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Inner Product
|
||||
static inline float InnerProductSSE(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum_0 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_1 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0);
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1);
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0);
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
xmm_sum_0 = _mm_add_epi32(
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0);
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1);
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0);
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
xmm_sum_0 = _mm_add_epi32(
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Inner Product
|
||||
static inline float InnerProductAVX(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
float result = 0.0;
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void InnerProductMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = InnerProductAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = InnerProductSSE(m, q, dim);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=1)
|
||||
void InnerProductMatrix<int8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_2X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=2)
|
||||
void InnerProductMatrix<int8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_2X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=1)
|
||||
void InnerProductMatrix<int8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X1_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=2)
|
||||
void InnerProductMatrix<int8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X2_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=4)
|
||||
void InnerProductMatrix<int8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X4_AVX(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_4X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=1)
|
||||
void InnerProductMatrix<int8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=2)
|
||||
void InnerProductMatrix<int8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=4)
|
||||
void InnerProductMatrix<int8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=8)
|
||||
void InnerProductMatrix<int8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_8X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=1)
|
||||
void InnerProductMatrix<int8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=2)
|
||||
void InnerProductMatrix<int8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=4)
|
||||
void InnerProductMatrix<int8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=8)
|
||||
void InnerProductMatrix<int8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=16)
|
||||
void InnerProductMatrix<int8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_16X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=1)
|
||||
void InnerProductMatrix<int8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X1_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X1_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=2)
|
||||
void InnerProductMatrix<int8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X2_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X2_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=4)
|
||||
void InnerProductMatrix<int8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X4_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X4_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=8)
|
||||
void InnerProductMatrix<int8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X8_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X8_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=16)
|
||||
void InnerProductMatrix<int8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X16_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X16_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=32)
|
||||
void InnerProductMatrix<int8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X32_AVX(m, q, dim, out, _mm256_cvtepi32_ps)
|
||||
#else
|
||||
ACCUM_INT8_32X32_SSE(m, q, dim, out, _mm_cvtepi32_ps)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = -InnerProductAVX(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = -InnerProductSSE(m, q, dim);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 2, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X1_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_2X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=2, N=2)
|
||||
void MinusInnerProductMatrix<int8_t, 2, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_2X2_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_2X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 4, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X1_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=2)
|
||||
void MinusInnerProductMatrix<int8_t, 4, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X2_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=4, N=4)
|
||||
void MinusInnerProductMatrix<int8_t, 4, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_4X4_AVX(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#else
|
||||
ACCUM_INT8_4X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 8, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=2)
|
||||
void MinusInnerProductMatrix<int8_t, 8, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=4)
|
||||
void MinusInnerProductMatrix<int8_t, 8, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=8, N=8)
|
||||
void MinusInnerProductMatrix<int8_t, 8, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_8X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_8X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 16, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=2)
|
||||
void MinusInnerProductMatrix<int8_t, 16, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=4)
|
||||
void MinusInnerProductMatrix<int8_t, 16, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=8)
|
||||
void MinusInnerProductMatrix<int8_t, 16, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=16, N=16)
|
||||
void MinusInnerProductMatrix<int8_t, 16, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_16X16_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_16X16_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X1_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X1_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=2)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 2>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X2_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X2_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=4)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 4>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X4_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X4_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=8)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 8>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X8_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X8_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=16)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 16>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X16_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X16_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=32, N=32)
|
||||
void MinusInnerProductMatrix<int8_t, 32, 32>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
ACCUM_INT8_32X32_AVX(m, q, dim, out, NEGATE_FP32_AVX)
|
||||
#else
|
||||
ACCUM_INT8_32X32_SSE(m, q, dim, out, NEGATE_FP32_SSE)
|
||||
#endif // __AVX2__
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,189 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Inner Product
|
||||
float InnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
float result = 0.0;
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
|
||||
ymm_lhs_0 = _mm256_sign_epi8(ymm_lhs_0, ymm_rhs_0);
|
||||
ymm_lhs_1 = _mm256_sign_epi8(ymm_lhs_1, ymm_rhs_1);
|
||||
ymm_rhs_0 = _mm256_abs_epi8(ymm_rhs_0);
|
||||
ymm_rhs_1 = _mm256_abs_epi8(ymm_rhs_1);
|
||||
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_0, ymm_lhs_0),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs_1, ymm_lhs_1),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
ymm_lhs = _mm256_sign_epi8(ymm_lhs, ymm_rhs);
|
||||
ymm_rhs = _mm256_abs_epi8(ymm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_madd_epi16(_mm256_maddubs_epi16(ymm_rhs, ymm_lhs),
|
||||
ONES_INT16_AVX),
|
||||
ymm_sum_0);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(),
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs),
|
||||
ONES_INT16_SSE)),
|
||||
ymm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
float MinusInnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size) {
|
||||
return -InnerProductAVX2(lhs, rhs, size);
|
||||
}
|
||||
|
||||
#endif // __AVX2__
|
||||
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
float InnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
float MinusInnerProductAVX2(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
float InnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
float MinusInnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void InnerProductMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = InnerProductAVX2(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = InnerProductSSE(m, q, dim);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (INT8, M=1, N=1)
|
||||
void MinusInnerProductMatrix<int8_t, 1, 1>::Compute(const ValueType *m,
|
||||
const ValueType *q,
|
||||
size_t dim, float *out) {
|
||||
#if defined(__AVX2__)
|
||||
if (dim > 31) {
|
||||
*out = MinusInnerProductAVX2(m, q, dim);
|
||||
return;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
*out = MinusInnerProductSSE(m, q, dim);
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,157 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_inner_product_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Inner Product
|
||||
float InnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum_0 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_1 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0);
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1);
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0);
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
xmm_sum_0 = _mm_add_epi32(
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
|
||||
xmm_lhs_0 = _mm_sign_epi8(xmm_lhs_0, xmm_rhs_0);
|
||||
xmm_lhs_1 = _mm_sign_epi8(xmm_lhs_1, xmm_rhs_1);
|
||||
xmm_rhs_0 = _mm_abs_epi8(xmm_rhs_0);
|
||||
xmm_rhs_1 = _mm_abs_epi8(xmm_rhs_1);
|
||||
xmm_sum_0 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_0, xmm_lhs_0),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
xmm_sum_1 =
|
||||
_mm_add_epi32(_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs_1, xmm_lhs_1),
|
||||
ONES_INT16_SSE),
|
||||
xmm_sum_1);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
|
||||
xmm_lhs = _mm_sign_epi8(xmm_lhs, xmm_rhs);
|
||||
xmm_rhs = _mm_abs_epi8(xmm_rhs);
|
||||
xmm_sum_0 = _mm_add_epi32(
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(xmm_rhs, xmm_lhs), ONES_INT16_SSE),
|
||||
xmm_sum_0);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V128(_mm_add_epi32(xmm_sum_0, xmm_sum_1)));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result)
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
float MinusInnerProductSSE(const int8_t *lhs, const int8_t *rhs, size_t size) {
|
||||
return -InnerProductSSE(lhs, rhs, size);
|
||||
}
|
||||
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,409 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX512)
|
||||
#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum);
|
||||
#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \
|
||||
zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask);
|
||||
|
||||
#define HorizontalAdd_FP16_NEON(v) \
|
||||
vaddvq_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(v)), vcvt_high_f32_f16(v)))
|
||||
|
||||
#define HorizontalAdd_FP32_V512_TO_V256(zmm) \
|
||||
_mm256_add_ps( \
|
||||
_mm512_castps512_ps256(zmm), \
|
||||
_mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(zmm), 1)))
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX, FP16)
|
||||
#define FMA_FP16_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
float v1 = lhs; \
|
||||
float v2 = rhs; \
|
||||
sum += v1 * v2; \
|
||||
norm1 += v1 * v1; \
|
||||
norm2 += v2 * v2; \
|
||||
}
|
||||
|
||||
#if defined(__ARM_NEON) && defined(__aarch64__)
|
||||
#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormNEON(const Float16 *lhs,
|
||||
const Float16 *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 3) << 3);
|
||||
float16x8_t v_sum = vdupq_n_f16(0);
|
||||
float16x8_t v_sum_norm1 = vdupq_n_f16(0);
|
||||
float16x8_t v_sum_norm2 = vdupq_n_f16(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs);
|
||||
float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs);
|
||||
v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs);
|
||||
v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs);
|
||||
v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float16x8_t v_lhs = vcombine_f16(vld1_f16((const float16_t *)lhs),
|
||||
vreinterpret_f16_u64(vdup_n_u64(0ul)));
|
||||
float16x8_t v_rhs = vcombine_f16(vld1_f16((const float16_t *)rhs),
|
||||
vreinterpret_f16_u64(vdup_n_u64(0ul)));
|
||||
v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs);
|
||||
v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs);
|
||||
v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP16_NEON(v_sum);
|
||||
float norm1 = HorizontalAdd_FP16_NEON(v_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP16_NEON(v_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#else
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormNEON(const Float16 *lhs,
|
||||
const Float16 *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 3) << 3);
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm2 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs);
|
||||
float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs);
|
||||
float32x4_t v_lhs_0 = vcvt_f32_f16(vget_low_f16(v_lhs));
|
||||
float32x4_t v_rhs_0 = vcvt_f32_f16(vget_low_f16(v_rhs));
|
||||
float32x4_t v_lhs_1 = vcvt_high_f32_f16(v_lhs);
|
||||
float32x4_t v_rhs_1 = vcvt_high_f32_f16(v_rhs);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_lhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)lhs));
|
||||
float32x4_t v_rhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)rhs));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
float norm1 = vaddvq_f32(v_sum_norm1);
|
||||
float norm2 = vaddvq_f32(v_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
|
||||
#endif // __ARM_NEON && __aarch64__
|
||||
|
||||
#if defined(__AVX__) && defined(__F16C__)
|
||||
#if defined(__AVX512F__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX512(const Float16 *lhs,
|
||||
const Float16 *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm2 = _mm512_setzero_ps();
|
||||
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 5) << 5);
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512i zmm_lhs = _mm512_load_si512((const __m512i *)lhs);
|
||||
__m512i zmm_rhs = _mm512_load_si512((const __m512i *)rhs);
|
||||
__m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs));
|
||||
__m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1));
|
||||
__m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs));
|
||||
__m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_load_si256((const __m256i *)lhs));
|
||||
__m512 zmm_rhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_load_si256((const __m256i *)rhs));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512i zmm_lhs = _mm512_loadu_si512((const __m512i *)lhs);
|
||||
__m512i zmm_rhs = _mm512_loadu_si512((const __m512i *)rhs);
|
||||
__m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs));
|
||||
__m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1));
|
||||
__m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs));
|
||||
__m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)lhs));
|
||||
__m512 zmm_rhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)rhs));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
__m256 ymm_sum_0 =
|
||||
HorizontalAdd_FP32_V512_TO_V256(_mm512_add_ps(zmm_sum_0, zmm_sum_1));
|
||||
__m256 ymm_sum_norm1 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm1);
|
||||
__m256 ymm_sum_norm2 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm2);
|
||||
if (last >= lhs + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP32_V256(ymm_sum_0);
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#else
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX(const Float16 *lhs,
|
||||
const Float16 *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm2 = _mm256_setzero_ps();
|
||||
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 4) << 4);
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs));
|
||||
__m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs));
|
||||
__m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs));
|
||||
__m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs));
|
||||
__m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
#endif // __AVX__ && __F16C__
|
||||
|
||||
#if (defined(__F16C__) && defined(__AVX__)) || \
|
||||
(defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
#elif defined(__AVX512F__)
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
#endif
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
#elif defined(__AVX512F__)
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
#endif
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,116 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__) && defined(__F16C__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm2 = _mm256_setzero_ps();
|
||||
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 4) << 4);
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs));
|
||||
__m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs));
|
||||
__m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_load_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_lhs));
|
||||
__m256 ymm_lhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_lhs, 1));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm256_castsi256_si128(ymm_rhs));
|
||||
__m256 ymm_rhs_1 = _mm256_cvtph_ps(_mm256_extractf128_si256(ymm_rhs, 1));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX__ && __F16C__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,134 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm2 = _mm512_setzero_ps();
|
||||
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 5) << 5);
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512i zmm_lhs = _mm512_load_si512((const __m512i *)lhs);
|
||||
__m512i zmm_rhs = _mm512_load_si512((const __m512i *)rhs);
|
||||
__m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs));
|
||||
__m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1));
|
||||
__m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs));
|
||||
__m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_load_si256((const __m256i *)lhs));
|
||||
__m512 zmm_rhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_load_si256((const __m256i *)rhs));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512i zmm_lhs = _mm512_loadu_si512((const __m512i *)lhs);
|
||||
__m512i zmm_rhs = _mm512_loadu_si512((const __m512i *)rhs);
|
||||
__m512 zmm_lhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_lhs));
|
||||
__m512 zmm_lhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_lhs, 1));
|
||||
__m512 zmm_rhs_0 = _mm512_cvtph_ps(_mm512_castsi512_si256(zmm_rhs));
|
||||
__m512 zmm_rhs_1 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(zmm_rhs, 1));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)lhs));
|
||||
__m512 zmm_rhs_0 =
|
||||
_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)rhs));
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
__m256 ymm_sum_0 =
|
||||
HorizontalAdd_FP32_V512_TO_V256(_mm512_add_ps(zmm_sum_0, zmm_sum_1));
|
||||
__m256 ymm_sum_norm1 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm1);
|
||||
__m256 ymm_sum_norm2 = HorizontalAdd_FP32_V512_TO_V256(zmm_sum_norm2);
|
||||
if (last >= lhs + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)lhs));
|
||||
__m256 ymm_rhs_0 = _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)rhs));
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP32_V256(ymm_sum_0);
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP16_GENERAL(lhs[6], rhs[6], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP16_GENERAL(lhs[5], rhs[5], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP16_GENERAL(lhs[4], rhs[4], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP16_GENERAL(lhs[3], rhs[3], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
float InnerProductAndSquaredNormAVX512(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
float InnerProductAndSquaredNormAVX(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if (defined(__F16C__) && defined(__AVX__)) || \
|
||||
(defined(__ARM_NEON) && defined(__aarch64__))
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif //__AVX512F__
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
}
|
||||
#endif //__ARM_NEON
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif //__AVX512F__
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
}
|
||||
#endif //__ARM_NEON
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
|
||||
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,126 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp16.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON) && defined(__aarch64__)
|
||||
#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 3) << 3);
|
||||
float16x8_t v_sum = vdupq_n_f16(0);
|
||||
float16x8_t v_sum_norm1 = vdupq_n_f16(0);
|
||||
float16x8_t v_sum_norm2 = vdupq_n_f16(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs);
|
||||
float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs);
|
||||
v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs);
|
||||
v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs);
|
||||
v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float16x8_t v_lhs = vcombine_f16(vld1_f16((const float16_t *)lhs),
|
||||
vreinterpret_f16_u64(vdup_n_u64(0ul)));
|
||||
float16x8_t v_rhs = vcombine_f16(vld1_f16((const float16_t *)rhs),
|
||||
vreinterpret_f16_u64(vdup_n_u64(0ul)));
|
||||
v_sum = vfmaq_f16(v_sum, v_lhs, v_rhs);
|
||||
v_sum_norm1 = vfmaq_f16(v_sum_norm1, v_lhs, v_lhs);
|
||||
v_sum_norm2 = vfmaq_f16(v_sum_norm2, v_rhs, v_rhs);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = HorizontalAdd_FP16_NEON(v_sum);
|
||||
float norm1 = HorizontalAdd_FP16_NEON(v_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP16_NEON(v_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#else
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormNEON(const Float16 *lhs, const Float16 *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const Float16 *last = lhs + size;
|
||||
const Float16 *last_aligned = lhs + ((size >> 3) << 3);
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm2 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float16x8_t v_lhs = vld1q_f16((const float16_t *)lhs);
|
||||
float16x8_t v_rhs = vld1q_f16((const float16_t *)rhs);
|
||||
float32x4_t v_lhs_0 = vcvt_f32_f16(vget_low_f16(v_lhs));
|
||||
float32x4_t v_rhs_0 = vcvt_f32_f16(vget_low_f16(v_rhs));
|
||||
float32x4_t v_lhs_1 = vcvt_high_f32_f16(v_lhs);
|
||||
float32x4_t v_rhs_1 = vcvt_high_f32_f16(v_rhs);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_lhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)lhs));
|
||||
float32x4_t v_rhs_0 = vcvt_f32_f16(vld1_f16((const float16_t *)rhs));
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
float norm1 = vaddvq_f32(v_sum_norm1);
|
||||
float norm2 = vaddvq_f32(v_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP16_GENERAL(lhs[2], rhs[2], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP16_GENERAL(lhs[1], rhs[1], result, norm1, norm2);
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP16_GENERAL(lhs[0], rhs[0], result, norm1, norm2);
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
|
||||
#endif // __ARM_NEON && __aarch64__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,684 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_FP32_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += (lhs) * (rhs); \
|
||||
norm1 += (lhs) * (lhs); \
|
||||
norm2 += (rhs) * (rhs); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX512)
|
||||
#define FMA_FP32_AVX512(zmm_m, zmm_q, zmm_sum) \
|
||||
zmm_sum = _mm512_fmadd_ps(zmm_m, zmm_q, zmm_sum);
|
||||
#define FMA_MASK_FP32_AVX512(zmm_m, zmm_q, zmm_sum, mask) \
|
||||
zmm_sum = _mm512_mask3_fmadd_ps(zmm_m, zmm_q, zmm_sum, mask);
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormNEON(const float *lhs,
|
||||
const float *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm2 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float32x4_t v_lhs_0 = vld1q_f32(lhs + 0);
|
||||
float32x4_t v_lhs_1 = vld1q_f32(lhs + 4);
|
||||
float32x4_t v_rhs_0 = vld1q_f32(rhs + 0);
|
||||
float32x4_t v_rhs_1 = vld1q_f32(rhs + 4);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_lhs_0 = vld1q_f32(lhs);
|
||||
float32x4_t v_rhs_0 = vld1q_f32(rhs);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
float norm1 = vaddvq_f32(v_sum_norm1);
|
||||
float norm2 = vaddvq_f32(v_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __ARM_NEON
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormSSE(const float *lhs,
|
||||
const float *rhs, size_t size,
|
||||
float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
__m128 xmm_sum = _mm_setzero_ps();
|
||||
__m128 xmm_sum_norm1 = _mm_setzero_ps();
|
||||
__m128 xmm_sum_norm2 = _mm_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_load_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_load_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_load_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_load_ps(rhs + 4);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_lhs_0 = _mm_load_ps(lhs);
|
||||
__m128 xmm_rhs_0 = _mm_load_ps(rhs);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_lhs_0 = _mm_loadu_ps(lhs);
|
||||
__m128 xmm_rhs_0 = _mm_loadu_ps(rhs);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V128(xmm_sum);
|
||||
float norm1 = HorizontalAdd_FP32_V128(xmm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V128(xmm_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE__
|
||||
|
||||
#if defined(__AVX__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX(const float *lhs,
|
||||
const float *rhs, size_t size,
|
||||
float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm2 = _mm256_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_load_ps(lhs);
|
||||
__m256 ymm_rhs_0 = _mm256_load_ps(rhs);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_loadu_ps(lhs);
|
||||
__m256 ymm_rhs_0 = _mm256_loadu_ps(rhs);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP32_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP32_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP32_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP32_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX512(const float *lhs,
|
||||
const float *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm2 = _mm512_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_lhs_0 = _mm512_load_ps(lhs + 0);
|
||||
__m512 zmm_lhs_1 = _mm512_load_ps(lhs + 16);
|
||||
__m512 zmm_rhs_0 = _mm512_load_ps(rhs + 0);
|
||||
__m512 zmm_rhs_1 = _mm512_load_ps(rhs + 16);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 = _mm512_load_ps(lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_load_ps(rhs);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_lhs_0 = _mm512_loadu_ps(lhs + 0);
|
||||
__m512 zmm_lhs_1 = _mm512_loadu_ps(lhs + 16);
|
||||
__m512 zmm_rhs_0 = _mm512_loadu_ps(rhs + 0);
|
||||
__m512 zmm_rhs_1 = _mm512_loadu_ps(rhs + 16);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 = _mm512_loadu_ps(lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_loadu_ps(rhs);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1);
|
||||
__m512 zmm_undefined = _mm512_undefined_ps();
|
||||
__m512 zmm_lhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, rhs);
|
||||
FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0, mask);
|
||||
FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1, mask);
|
||||
FMA_MASK_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2, mask);
|
||||
}
|
||||
|
||||
*sql = HorizontalAdd_FP32_V512(zmm_sum_norm1);
|
||||
*sqr = HorizontalAdd_FP32_V512(zmm_sum_norm2);
|
||||
return HorizontalAdd_FP32_V512(zmm_sum_0);
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
if (dim > 15) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (dim > 7) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX__
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
if (dim > 15) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (dim > 7) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX__
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // __SSE__
|
||||
|
||||
// #if 1
|
||||
#if defined(__SSE4_1__)
|
||||
const static __m128i SHUFFLE_MASK16[16] = {
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, -127, -127),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5,
|
||||
4),
|
||||
_mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
};
|
||||
|
||||
constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536;
|
||||
|
||||
float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
// size_t alloc_size = 0;
|
||||
|
||||
size_t i1 = 0, i2 = 0;
|
||||
size_t end1 = m_sparse_count / 8 * 8;
|
||||
size_t end2 = q_sparse_count / 8 * 8;
|
||||
|
||||
// std::vector<float> mem1;
|
||||
// std::vector<float> mem2;
|
||||
|
||||
float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH];
|
||||
float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH];
|
||||
|
||||
float *val_start_1 = fixed_buffer_1;
|
||||
float *val_start_2 = fixed_buffer_2;
|
||||
|
||||
// uint32_t max_count = std::max(m_sparse_count, q_sparse_count);
|
||||
|
||||
// if (MAX_SPARSE_BUFFER_LENGTH < max_count) {
|
||||
// mem1.reserve(max_count);
|
||||
// mem2.reserve(max_count);
|
||||
|
||||
// val_start_1 = mem1.data();
|
||||
// val_start_2 = mem2.data();
|
||||
// }
|
||||
|
||||
float *val_1 = val_start_1;
|
||||
float *val_2 = val_start_2;
|
||||
|
||||
if (i1 < end1 && i2 < end2) {
|
||||
while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
}
|
||||
|
||||
while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
}
|
||||
|
||||
__m128i mm_index_m =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
__m128i mm_index_q =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
|
||||
while (true) {
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "index 1: " << std::endl;
|
||||
print_data16(&mm_index_m);
|
||||
|
||||
std::cout << "index 2: " << std::endl;
|
||||
print_data16(&mm_index_q);
|
||||
#endif
|
||||
|
||||
__m128i mm_cmp_res =
|
||||
_mm_cmpistrm(mm_index_q, mm_index_m,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "cmp res: " << std::endl;
|
||||
print_data16(&mm_cmp_res);
|
||||
#endif
|
||||
|
||||
int r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
if (r) {
|
||||
int r1 = r & 15;
|
||||
|
||||
__m128i v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1]));
|
||||
__m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r1);
|
||||
|
||||
int r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r2);
|
||||
|
||||
mm_cmp_res = _mm_cmpistrm(
|
||||
mm_index_m, mm_index_q,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
r1 = r & 15;
|
||||
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r1);
|
||||
|
||||
r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r2);
|
||||
}
|
||||
|
||||
const uint16_t id1_max = m_sparse_index[i1 + 7];
|
||||
|
||||
if (id1_max <= q_sparse_index[i2 + 7]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
mm_index_m = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
}
|
||||
|
||||
if (id1_max >= q_sparse_index[i2 + 7]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
mm_index_q = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_scalar:
|
||||
while (i1 < m_sparse_count && i2 < q_sparse_count) {
|
||||
if (m_sparse_index[i1] == q_sparse_index[i2]) {
|
||||
*val_1++ = m_sparse_value[i1];
|
||||
*val_2++ = q_sparse_value[i2];
|
||||
|
||||
++i1;
|
||||
++i2;
|
||||
} else if (m_sparse_index[i1] < q_sparse_index[i2]) {
|
||||
++i1;
|
||||
} else {
|
||||
++i2;
|
||||
}
|
||||
}
|
||||
|
||||
size_t res_num = val_1 - val_start_1;
|
||||
|
||||
// if (res_num != val_2 - val_start_2) {
|
||||
// std::cerr << "size mismatch!" << std::endl;
|
||||
// }
|
||||
|
||||
size_t res_num4 = res_num / 4 * 4;
|
||||
|
||||
if (res_num4) {
|
||||
__m128 sum128 = _mm_set1_ps(0);
|
||||
|
||||
for (size_t k = 0; k < res_num4; k += 4) {
|
||||
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k),
|
||||
_mm_loadu_ps(val_start_2 + k)));
|
||||
}
|
||||
|
||||
float __attribute__((aligned(16))) tmp_res[4];
|
||||
_mm_store_ps(tmp_res, sum128);
|
||||
sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]);
|
||||
}
|
||||
|
||||
for (size_t k = res_num4; k < res_num; ++k)
|
||||
sum += val_start_1[k] * val_start_2[k];
|
||||
|
||||
return sum;
|
||||
}
|
||||
#else
|
||||
float MipsInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
size_t m_i = 0;
|
||||
size_t q_i = 0;
|
||||
while (m_i < m_sparse_count && q_i < q_sparse_count) {
|
||||
if (m_sparse_index[m_i] == q_sparse_index[q_i]) {
|
||||
sum += m_sparse_value[m_i] * q_sparse_value[q_i];
|
||||
|
||||
++m_i;
|
||||
++q_i;
|
||||
} else if (m_sparse_index[m_i] < q_sparse_index[q_i]) {
|
||||
++m_i;
|
||||
} else {
|
||||
++q_i;
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
template <>
|
||||
float MipsSquaredEuclideanSparseDistanceMatrix<float>::
|
||||
ComputeInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const ValueType *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const ValueType *q_sparse_value) {
|
||||
#if defined(__SSE4_1__)
|
||||
return MipsInnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#else
|
||||
return MipsInnerProductSparseInSegment(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif //__ARM_NEON
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,114 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 4) << 4);
|
||||
|
||||
__m256 ymm_sum_0 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm1 = _mm256_setzero_ps();
|
||||
__m256 ymm_sum_norm2 = _mm256_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_load_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_load_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_load_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_load_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_load_ps(lhs);
|
||||
__m256 ymm_rhs_0 = _mm256_load_ps(rhs);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m256 ymm_lhs_0 = _mm256_loadu_ps(lhs + 0);
|
||||
__m256 ymm_lhs_1 = _mm256_loadu_ps(lhs + 8);
|
||||
__m256 ymm_rhs_0 = _mm256_loadu_ps(rhs + 0);
|
||||
__m256 ymm_rhs_1 = _mm256_loadu_ps(rhs + 8);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 8) {
|
||||
__m256 ymm_lhs_0 = _mm256_loadu_ps(lhs);
|
||||
__m256 ymm_rhs_0 = _mm256_loadu_ps(rhs);
|
||||
ymm_sum_0 = _mm256_fmadd_ps(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_fmadd_ps(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_fmadd_ps(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
lhs += 8;
|
||||
rhs += 8;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V256(_mm256_add_ps(ymm_sum_0, ymm_sum_1));
|
||||
float norm1 = HorizontalAdd_FP32_V256(ymm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V256(ymm_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 7:
|
||||
FMA_FP32_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_FP32_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_FP32_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_FP32_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,100 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX512(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m512 zmm_sum_0 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm1 = _mm512_setzero_ps();
|
||||
__m512 zmm_sum_norm2 = _mm512_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0x3f) == 0 && ((uintptr_t)rhs & 0x3f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_lhs_0 = _mm512_load_ps(lhs + 0);
|
||||
__m512 zmm_lhs_1 = _mm512_load_ps(lhs + 16);
|
||||
__m512 zmm_rhs_0 = _mm512_load_ps(rhs + 0);
|
||||
__m512 zmm_rhs_1 = _mm512_load_ps(rhs + 16);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 = _mm512_load_ps(lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_load_ps(rhs);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m512 zmm_lhs_0 = _mm512_loadu_ps(lhs + 0);
|
||||
__m512 zmm_lhs_1 = _mm512_loadu_ps(lhs + 16);
|
||||
__m512 zmm_rhs_0 = _mm512_loadu_ps(rhs + 0);
|
||||
__m512 zmm_rhs_1 = _mm512_loadu_ps(rhs + 16);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_rhs_1, zmm_sum_1)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_lhs_1, zmm_lhs_1, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
FMA_FP32_AVX512(zmm_rhs_1, zmm_rhs_1, zmm_sum_norm2)
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m512 zmm_lhs_0 = _mm512_loadu_ps(lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_loadu_ps(rhs);
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0)
|
||||
FMA_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1)
|
||||
FMA_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2)
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
|
||||
zmm_sum_0 = _mm512_add_ps(zmm_sum_0, zmm_sum_1);
|
||||
if (lhs != last) {
|
||||
__mmask16 mask = (__mmask16)((1 << (last - lhs)) - 1);
|
||||
__m512 zmm_undefined = _mm512_undefined_ps();
|
||||
__m512 zmm_lhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, lhs);
|
||||
__m512 zmm_rhs_0 = _mm512_mask_loadu_ps(zmm_undefined, mask, rhs);
|
||||
FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_rhs_0, zmm_sum_0, mask);
|
||||
FMA_MASK_FP32_AVX512(zmm_lhs_0, zmm_lhs_0, zmm_sum_norm1, mask);
|
||||
FMA_MASK_FP32_AVX512(zmm_rhs_0, zmm_rhs_0, zmm_sum_norm2, mask);
|
||||
}
|
||||
|
||||
*sql = HorizontalAdd_FP32_V512(zmm_sum_norm1);
|
||||
*sqr = HorizontalAdd_FP32_V512(zmm_sum_norm2);
|
||||
return HorizontalAdd_FP32_V512(zmm_sum_0);
|
||||
}
|
||||
#endif // __AVX512F__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
float InnerProductAndSquaredNormNEON(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
float InnerProductAndSquaredNormAVX512(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX__)
|
||||
float InnerProductAndSquaredNormAVX(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__)
|
||||
float InnerProductAndSquaredNormSSE(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value);
|
||||
#endif
|
||||
|
||||
float MipsInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value);
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F && dim > 15) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX && dim > 7) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX__
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F && dim > 15) {
|
||||
sum = InnerProductAndSquaredNormAVX512(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX512F__
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX && dim > 7) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif // __AVX__
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // __SSE__
|
||||
|
||||
template <>
|
||||
float MipsSquaredEuclideanSparseDistanceMatrix<float>::
|
||||
ComputeInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const ValueType *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const ValueType *q_sparse_value) {
|
||||
#if defined(__SSE4_1__)
|
||||
return MipsInnerProductSparseInSegmentSSE(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#else
|
||||
return MipsInnerProductSparseInSegment(m_sparse_count, m_sparse_index,
|
||||
m_sparse_value, q_sparse_count,
|
||||
q_sparse_index, q_sparse_value);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,105 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__ARM_NEON)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormNEON(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
float32x4_t v_sum_0 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm1 = vdupq_n_f32(0);
|
||||
float32x4_t v_sum_norm2 = vdupq_n_f32(0);
|
||||
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
float32x4_t v_lhs_0 = vld1q_f32(lhs + 0);
|
||||
float32x4_t v_lhs_1 = vld1q_f32(lhs + 4);
|
||||
float32x4_t v_rhs_0 = vld1q_f32(rhs + 0);
|
||||
float32x4_t v_rhs_1 = vld1q_f32(rhs + 4);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_1 = vfmaq_f32(v_sum_1, v_lhs_1, v_rhs_1);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_1, v_lhs_1);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_1, v_rhs_1);
|
||||
}
|
||||
if (last >= last_aligned + 4) {
|
||||
float32x4_t v_lhs_0 = vld1q_f32(lhs);
|
||||
float32x4_t v_rhs_0 = vld1q_f32(rhs);
|
||||
v_sum_0 = vfmaq_f32(v_sum_0, v_lhs_0, v_rhs_0);
|
||||
v_sum_norm1 = vfmaq_f32(v_sum_norm1, v_lhs_0, v_lhs_0);
|
||||
v_sum_norm2 = vfmaq_f32(v_sum_norm2, v_rhs_0, v_rhs_0);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
|
||||
float result = vaddvq_f32(vaddq_f32(v_sum_0, v_sum_1));
|
||||
float norm1 = vaddvq_f32(v_sum_norm1);
|
||||
float norm2 = vaddvq_f32(v_sum_norm2);
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum = InnerProductAndSquaredNormNEON(p, q, dim, &u2, &v2);
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif //__ARM_NEON
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,336 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_fp32.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormSSE(const float *lhs, const float *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const float *last = lhs + size;
|
||||
const float *last_aligned = lhs + ((size >> 3) << 3);
|
||||
|
||||
__m128 xmm_sum = _mm_setzero_ps();
|
||||
__m128 xmm_sum_norm1 = _mm_setzero_ps();
|
||||
__m128 xmm_sum_norm2 = _mm_setzero_ps();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_load_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_load_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_load_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_load_ps(rhs + 4);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_lhs_0 = _mm_load_ps(lhs);
|
||||
__m128 xmm_rhs_0 = _mm_load_ps(rhs);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 8, rhs += 8) {
|
||||
__m128 xmm_lhs_0 = _mm_loadu_ps(lhs + 0);
|
||||
__m128 xmm_lhs_1 = _mm_loadu_ps(lhs + 4);
|
||||
__m128 xmm_rhs_0 = _mm_loadu_ps(rhs + 0);
|
||||
__m128 xmm_rhs_1 = _mm_loadu_ps(rhs + 4);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 4) {
|
||||
__m128 xmm_lhs_0 = _mm_loadu_ps(lhs);
|
||||
__m128 xmm_rhs_0 = _mm_loadu_ps(rhs);
|
||||
xmm_sum = _mm_fmadd_ps(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
xmm_sum_norm1 = _mm_fmadd_ps(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
xmm_sum_norm2 = _mm_fmadd_ps(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
lhs += 4;
|
||||
rhs += 4;
|
||||
}
|
||||
}
|
||||
float result = HorizontalAdd_FP32_V128(xmm_sum);
|
||||
float norm1 = HorizontalAdd_FP32_V128(xmm_sum_norm1);
|
||||
float norm2 = HorizontalAdd_FP32_V128(xmm_sum_norm2);
|
||||
|
||||
switch (last - lhs) {
|
||||
case 3:
|
||||
FMA_FP32_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_FP32_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_FP32_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // __SSE__
|
||||
|
||||
// #if 1
|
||||
#if defined(__SSE4_1__)
|
||||
const static __m128i SHUFFLE_MASK16[16] = {
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, -127, -127, -127, -127),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 7, 6, 5, 4, 3,
|
||||
2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 11, 10, 9, 8,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, -127, -127,
|
||||
-127, -127, 15, 14, 13, 12),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
3, 2, 1, 0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
7, 6, 5, 4),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 7, 6, 5, 4, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, -127, -127, -127, -127, 15, 14, 13, 12,
|
||||
11, 10, 9, 8),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 3, 2, 1,
|
||||
0),
|
||||
_mm_set_epi8(-127, -127, -127, -127, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5,
|
||||
4),
|
||||
_mm_set_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0),
|
||||
};
|
||||
|
||||
constexpr uint32_t MAX_SPARSE_BUFFER_LENGTH = 65536;
|
||||
|
||||
float MipsInnerProductSparseInSegmentSSE(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
// size_t alloc_size = 0;
|
||||
|
||||
size_t i1 = 0, i2 = 0;
|
||||
size_t end1 = m_sparse_count / 8 * 8;
|
||||
size_t end2 = q_sparse_count / 8 * 8;
|
||||
|
||||
// std::vector<float> mem1;
|
||||
// std::vector<float> mem2;
|
||||
|
||||
float fixed_buffer_1[MAX_SPARSE_BUFFER_LENGTH];
|
||||
float fixed_buffer_2[MAX_SPARSE_BUFFER_LENGTH];
|
||||
|
||||
float *val_start_1 = fixed_buffer_1;
|
||||
float *val_start_2 = fixed_buffer_2;
|
||||
|
||||
// uint32_t max_count = std::max(m_sparse_count, q_sparse_count);
|
||||
|
||||
// if (MAX_SPARSE_BUFFER_LENGTH < max_count) {
|
||||
// mem1.reserve(max_count);
|
||||
// mem2.reserve(max_count);
|
||||
|
||||
// val_start_1 = mem1.data();
|
||||
// val_start_2 = mem2.data();
|
||||
// }
|
||||
|
||||
float *val_1 = val_start_1;
|
||||
float *val_2 = val_start_2;
|
||||
|
||||
if (i1 < end1 && i2 < end2) {
|
||||
while (m_sparse_index[i1 + 7] < q_sparse_index[i2]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
}
|
||||
|
||||
while (q_sparse_index[i2 + 7] < m_sparse_index[i1]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
}
|
||||
|
||||
__m128i mm_index_m =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
__m128i mm_index_q =
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
|
||||
while (true) {
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "index 1: " << std::endl;
|
||||
print_data16(&mm_index_m);
|
||||
|
||||
std::cout << "index 2: " << std::endl;
|
||||
print_data16(&mm_index_q);
|
||||
#endif
|
||||
|
||||
__m128i mm_cmp_res =
|
||||
_mm_cmpistrm(mm_index_q, mm_index_m,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
|
||||
#ifdef DEBUG_PRINT
|
||||
std::cout << "cmp res: " << std::endl;
|
||||
print_data16(&mm_cmp_res);
|
||||
#endif
|
||||
|
||||
int r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
if (r) {
|
||||
int r1 = r & 15;
|
||||
|
||||
__m128i v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1]));
|
||||
__m128 vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r1);
|
||||
|
||||
int r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_value[i1 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_1, vs);
|
||||
val_1 += _mm_popcnt_u32(r2);
|
||||
|
||||
mm_cmp_res = _mm_cmpistrm(
|
||||
mm_index_m, mm_index_q,
|
||||
_SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK);
|
||||
r = _mm_extract_epi32(mm_cmp_res, 0);
|
||||
|
||||
r1 = r & 15;
|
||||
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r1]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r1);
|
||||
|
||||
r2 = (r >> 4) & 15;
|
||||
v = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_value[i2 + 4]));
|
||||
vs = _mm_castsi128_ps(_mm_shuffle_epi8(v, SHUFFLE_MASK16[r2]));
|
||||
_mm_storeu_ps(val_2, vs);
|
||||
val_2 += _mm_popcnt_u32(r2);
|
||||
}
|
||||
|
||||
const uint16_t id1_max = m_sparse_index[i1 + 7];
|
||||
|
||||
if (id1_max <= q_sparse_index[i2 + 7]) {
|
||||
i1 += 8;
|
||||
if (i1 >= end1) goto do_scalar;
|
||||
mm_index_m = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&m_sparse_index[i1]));
|
||||
}
|
||||
|
||||
if (id1_max >= q_sparse_index[i2 + 7]) {
|
||||
i2 += 8;
|
||||
if (i2 >= end2) goto do_scalar;
|
||||
mm_index_q = _mm_loadu_si128(
|
||||
reinterpret_cast<const __m128i *>(&q_sparse_index[i2]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
do_scalar:
|
||||
while (i1 < m_sparse_count && i2 < q_sparse_count) {
|
||||
if (m_sparse_index[i1] == q_sparse_index[i2]) {
|
||||
*val_1++ = m_sparse_value[i1];
|
||||
*val_2++ = q_sparse_value[i2];
|
||||
|
||||
++i1;
|
||||
++i2;
|
||||
} else if (m_sparse_index[i1] < q_sparse_index[i2]) {
|
||||
++i1;
|
||||
} else {
|
||||
++i2;
|
||||
}
|
||||
}
|
||||
|
||||
size_t res_num = val_1 - val_start_1;
|
||||
|
||||
// if (res_num != val_2 - val_start_2) {
|
||||
// std::cerr << "size mismatch!" << std::endl;
|
||||
// }
|
||||
|
||||
size_t res_num4 = res_num / 4 * 4;
|
||||
|
||||
if (res_num4) {
|
||||
__m128 sum128 = _mm_set1_ps(0);
|
||||
|
||||
for (size_t k = 0; k < res_num4; k += 4) {
|
||||
sum128 = _mm_add_ps(sum128, _mm_mul_ps(_mm_loadu_ps(val_start_1 + k),
|
||||
_mm_loadu_ps(val_start_2 + k)));
|
||||
}
|
||||
|
||||
float __attribute__((aligned(16))) tmp_res[4];
|
||||
_mm_store_ps(tmp_res, sum128);
|
||||
sum += (tmp_res[0] + tmp_res[1] + tmp_res[2] + tmp_res[3]);
|
||||
}
|
||||
|
||||
for (size_t k = res_num4; k < res_num; ++k)
|
||||
sum += val_start_1[k] * val_start_2[k];
|
||||
|
||||
return sum;
|
||||
}
|
||||
#else
|
||||
float MipsInnerProductSparseInSegment(uint32_t m_sparse_count,
|
||||
const uint16_t *m_sparse_index,
|
||||
const float *m_sparse_value,
|
||||
uint32_t q_sparse_count,
|
||||
const uint16_t *q_sparse_index,
|
||||
const float *q_sparse_value) {
|
||||
float sum = 0.0f;
|
||||
|
||||
size_t m_i = 0;
|
||||
size_t q_i = 0;
|
||||
while (m_i < m_sparse_count && q_i < q_sparse_count) {
|
||||
if (m_sparse_index[m_i] == q_sparse_index[q_i]) {
|
||||
sum += m_sparse_value[m_i] * q_sparse_value[q_i];
|
||||
|
||||
++m_i;
|
||||
++q_i;
|
||||
} else if (m_sparse_index[m_i] < q_sparse_index[q_i]) {
|
||||
++m_i;
|
||||
} else {
|
||||
++q_i;
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,358 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "inner_product_matrix.h"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
#include "norm_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Four-bits Convert Table
|
||||
static const AILEGO_ALIGNED(32) int8_t Int4ConvertTable[32] = {
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1,
|
||||
0, 1, 2, 3, 4, 5, 6, 7, -8, -7, -6, -5, -4, -3, -2, -1};
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i MASK_INT4_SSE = _mm_set1_epi32(0x0f0f0f0f);
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
static const __m128i INT4_LOOKUP_SSE =
|
||||
_mm_load_si128((const __m128i *)Int4ConvertTable);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i MASK_INT4_AVX = _mm256_set1_epi32(0x0f0f0f0f);
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
static const __m256i INT4_LOOKUP_AVX =
|
||||
_mm256_load_si256((const __m256i *)Int4ConvertTable);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT4_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += Int4MulTable[(((lhs) << 4) & 0xf0) | (((rhs) >> 0) & 0xf)] + \
|
||||
Int4MulTable[(((lhs) >> 0) & 0xf0) | (((rhs) >> 4) & 0xf)]; \
|
||||
norm1 += static_cast<float>( \
|
||||
((int8_t)((lhs) << 4) >> 4) * ((int8_t)((lhs) << 4) >> 4) + \
|
||||
((int8_t)((lhs) & 0xf0) >> 4) * ((int8_t)((lhs) & 0xf0) >> 4)); \
|
||||
norm2 += static_cast<float>( \
|
||||
((int8_t)((rhs) << 4) >> 4) * ((int8_t)((rhs) << 4) >> 4) + \
|
||||
((int8_t)((rhs) & 0xf0) >> 4) * ((int8_t)((rhs) & 0xf0) >> 4)); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum)
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum)
|
||||
|
||||
//! Compute the distance between matrix and query (SSE)
|
||||
#define FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum_0, xmm_sum_norm1, \
|
||||
xmm_sum_norm2) \
|
||||
{ \
|
||||
__m128i xmm_lhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_lhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_0 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, _mm_and_si128((xmm_rhs), MASK_INT4_SSE)); \
|
||||
__m128i xmm_lhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_lhs), 4), MASK_INT4_SSE)); \
|
||||
__m128i xmm_rhs_1 = _mm_shuffle_epi8( \
|
||||
INT4_LOOKUP_SSE, \
|
||||
_mm_and_si128(_mm_srli_epi32((xmm_rhs), 4), MASK_INT4_SSE)); \
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum_0); \
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1); \
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2); \
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum_0); \
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1); \
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2); \
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query (AVX)
|
||||
#define FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, \
|
||||
ymm_sum_norm1, ymm_sum_norm2) \
|
||||
{ \
|
||||
__m256i ymm_lhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_lhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_0 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, _mm256_and_si256((ymm_rhs), MASK_INT4_AVX)); \
|
||||
__m256i ymm_lhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_lhs), 4), MASK_INT4_AVX)); \
|
||||
__m256i ymm_rhs_1 = _mm256_shuffle_epi8( \
|
||||
INT4_LOOKUP_AVX, \
|
||||
_mm256_and_si256(_mm256_srli_epi32((ymm_rhs), 4), MASK_INT4_AVX)); \
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0); \
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1); \
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1); \
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1); \
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2); \
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2); \
|
||||
}
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX(const uint8_t *lhs,
|
||||
const uint8_t *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm2 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1,
|
||||
ymm_sum_norm2)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1,
|
||||
ymm_sum_norm2)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#else
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormSSE(const uint8_t *lhs,
|
||||
const uint8_t *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX2__)
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2);
|
||||
#endif
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX2__)
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2);
|
||||
#endif
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
#include "norm_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm2 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1,
|
||||
ymm_sum_norm2)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)(lhs));
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)(rhs));
|
||||
FMA_INT4_ITER_AVX(ymm_lhs, ymm_rhs, ymm_sum_0, ymm_sum1, ymm_sum_norm1,
|
||||
ymm_sum_norm2)
|
||||
}
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
ymm_sum_0 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum), ymm_sum_0);
|
||||
ymm_sum_norm1 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm1), ymm_sum_norm1);
|
||||
ymm_sum_norm2 = _mm256_add_epi32(
|
||||
_mm256_set_m128i(_mm_setzero_si128(), xmm_sum_norm2), ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,83 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "inner_product_matrix.h"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
#include "norm_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX__)
|
||||
float InnerProductAndSquaredNormAVX(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__)
|
||||
float InnerProductAndSquaredNormSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2);
|
||||
}
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<uint8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim >> 1, &u2, &v2);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim >> 1, &u2, &v2);
|
||||
}
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,104 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "inner_product_matrix.h"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
#include "norm_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormSSE(const uint8_t *lhs, const uint8_t *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const uint8_t *last = lhs + size;
|
||||
const uint8_t *last_aligned = lhs + ((size >> 4) << 4);
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 16, rhs += 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)(lhs));
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)(rhs));
|
||||
FMA_INT4_ITER_SSE(xmm_lhs, xmm_rhs, xmm_sum, xmm_sum_norm1, xmm_sum_norm2)
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT4_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT4_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT4_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT4_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT4_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT4_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT4_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT4_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT4_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT4_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT4_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT4_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT4_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT4_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT4_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -1,358 +0,0 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
static const __m128i ONES_INT16_SSE = _mm_set1_epi32(0x00010001);
|
||||
#endif // __SSE4_1__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
static const __m256i ONES_INT16_AVX = _mm256_set1_epi32(0x00010001);
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Calculate Fused-Multiply-Add (GENERAL)
|
||||
#define FMA_INT8_GENERAL(lhs, rhs, sum, norm1, norm2) \
|
||||
{ \
|
||||
sum += static_cast<float>(lhs * rhs); \
|
||||
norm1 += static_cast<float>(lhs * lhs); \
|
||||
norm2 += static_cast<float>(rhs * rhs); \
|
||||
}
|
||||
|
||||
//! Calculate Fused-Multiply-Add (AVX)
|
||||
#define FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_madd_epi16( \
|
||||
_mm256_maddubs_epi16(_mm256_abs_epi8(ymm_rhs), \
|
||||
_mm256_sign_epi8(ymm_lhs, ymm_rhs)), \
|
||||
ONES_INT16_AVX), \
|
||||
ymm_sum)
|
||||
#define FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum) \
|
||||
ymm_sum = _mm256_add_epi32( \
|
||||
_mm256_set_m128i( \
|
||||
_mm_setzero_si128(), \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE)), \
|
||||
ymm_sum)
|
||||
|
||||
//! Calculate Fused-Multiply-Add (SSE)
|
||||
#define FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum) \
|
||||
xmm_sum = _mm_add_epi32( \
|
||||
_mm_madd_epi16(_mm_maddubs_epi16(_mm_abs_epi8(xmm_rhs), \
|
||||
_mm_sign_epi8(xmm_lhs, xmm_rhs)), \
|
||||
ONES_INT16_SSE), \
|
||||
xmm_sum)
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormAVX(const int8_t *lhs,
|
||||
const int8_t *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm2 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#else
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
static inline float InnerProductAndSquaredNormSSE(const int8_t *lhs,
|
||||
const int8_t *rhs,
|
||||
size_t size, float *sql,
|
||||
float *sqr) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX2__)
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
#endif
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2;
|
||||
float v2;
|
||||
float sum;
|
||||
|
||||
#if defined(__AVX2__)
|
||||
sum = InnerProductAndSquaredNormAVX(p, q, dim, &u2, &v2);
|
||||
#else
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
#endif
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,159 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormAVX2(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 6) << 6);
|
||||
|
||||
__m256i ymm_sum_0 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm1 = _mm256_setzero_si256();
|
||||
__m256i ymm_sum_norm2 = _mm256_setzero_si256();
|
||||
|
||||
if (((uintptr_t)lhs & 0x1f) == 0 && ((uintptr_t)rhs & 0x1f) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_load_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_load_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_load_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_load_si256((const __m256i *)(rhs + 32));
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_load_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_load_si256((const __m256i *)rhs);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 64, rhs += 64) {
|
||||
__m256i ymm_lhs_0 = _mm256_loadu_si256((const __m256i *)(lhs + 0));
|
||||
__m256i ymm_lhs_1 = _mm256_loadu_si256((const __m256i *)(lhs + 32));
|
||||
__m256i ymm_rhs_0 = _mm256_loadu_si256((const __m256i *)(rhs + 0));
|
||||
__m256i ymm_rhs_1 = _mm256_loadu_si256((const __m256i *)(rhs + 32));
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_rhs_0, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_rhs_1, ymm_sum_1);
|
||||
FMA_INT8_AVX(ymm_lhs_0, ymm_lhs_0, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_lhs_1, ymm_lhs_1, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs_0, ymm_rhs_0, ymm_sum_norm2);
|
||||
FMA_INT8_AVX(ymm_rhs_1, ymm_rhs_1, ymm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 32) {
|
||||
__m256i ymm_lhs = _mm256_loadu_si256((const __m256i *)lhs);
|
||||
__m256i ymm_rhs = _mm256_loadu_si256((const __m256i *)rhs);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX(ymm_lhs, ymm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX(ymm_rhs, ymm_rhs, ymm_sum_norm2);
|
||||
lhs += 32;
|
||||
rhs += 32;
|
||||
}
|
||||
|
||||
if (last >= lhs + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_rhs, ymm_sum_0);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_lhs, xmm_lhs, ymm_sum_norm1);
|
||||
FMA_INT8_AVX_SSE_HYBRID(xmm_rhs, xmm_rhs, ymm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(
|
||||
HorizontalAdd_INT32_V256(_mm256_add_epi32(ymm_sum_0, ymm_sum_1)));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V256(ymm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __AVX2__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,81 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
float InnerProductAndSquaredNormAVX2(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE__)
|
||||
float InnerProductAndSquaredNormSSE(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size, float *sql, float *sqr);
|
||||
#endif
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the distance between matrix and query by SphericalInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, float e2, float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
sum = InnerProductAndSquaredNormAVX2(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
*out = ComputeSphericalInjection(sum, u2, v2, e2);
|
||||
}
|
||||
|
||||
//! Compute the distance between matrix and query by RepeatedQuadraticInjection
|
||||
void MipsSquaredEuclideanDistanceMatrix<int8_t, 1, 1>::Compute(
|
||||
const ValueType *p, const ValueType *q, size_t dim, size_t m, float e2,
|
||||
float *out) {
|
||||
float u2{0.0f};
|
||||
float v2{0.0f};
|
||||
float sum{0.0f};
|
||||
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
sum = InnerProductAndSquaredNormAVX2(p, q, dim, &u2, &v2);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
sum = InnerProductAndSquaredNormSSE(p, q, dim, &u2, &v2);
|
||||
}
|
||||
|
||||
sum = e2 * (u2 + v2 - 2 * sum);
|
||||
u2 *= e2;
|
||||
v2 *= e2;
|
||||
for (size_t i = 0; i < m; ++i) {
|
||||
sum += (u2 - v2) * (u2 - v2);
|
||||
u2 = u2 * u2;
|
||||
v2 = v2 * v2;
|
||||
}
|
||||
*out = sum;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,137 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "distance_matrix_accum_int8.i"
|
||||
#include "distance_matrix_mips_utility.i"
|
||||
#include "mips_euclidean_distance_matrix.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
#if defined(__SSE4_1__)
|
||||
//! Compute the Inner Product between p and q, and each Squared L2-Norm value
|
||||
float InnerProductAndSquaredNormSSE(const int8_t *lhs, const int8_t *rhs,
|
||||
size_t size, float *sql, float *sqr) {
|
||||
const int8_t *last = lhs + size;
|
||||
const int8_t *last_aligned = lhs + ((size >> 5) << 5);
|
||||
|
||||
__m128i xmm_sum = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm1 = _mm_setzero_si128();
|
||||
__m128i xmm_sum_norm2 = _mm_setzero_si128();
|
||||
|
||||
if (((uintptr_t)lhs & 0xf) == 0 && ((uintptr_t)rhs & 0xf) == 0) {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_load_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_load_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_load_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_load_si128((const __m128i *)(rhs + 16));
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_load_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_load_si128((const __m128i *)rhs);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
} else {
|
||||
for (; lhs != last_aligned; lhs += 32, rhs += 32) {
|
||||
__m128i xmm_lhs_0 = _mm_loadu_si128((const __m128i *)(lhs + 0));
|
||||
__m128i xmm_lhs_1 = _mm_loadu_si128((const __m128i *)(lhs + 16));
|
||||
__m128i xmm_rhs_0 = _mm_loadu_si128((const __m128i *)(rhs + 0));
|
||||
__m128i xmm_rhs_1 = _mm_loadu_si128((const __m128i *)(rhs + 16));
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_rhs_0, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_0, xmm_lhs_0, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_0, xmm_rhs_0, xmm_sum_norm2);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_rhs_1, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs_1, xmm_lhs_1, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs_1, xmm_rhs_1, xmm_sum_norm2);
|
||||
}
|
||||
|
||||
if (last >= last_aligned + 16) {
|
||||
__m128i xmm_lhs = _mm_loadu_si128((const __m128i *)lhs);
|
||||
__m128i xmm_rhs = _mm_loadu_si128((const __m128i *)rhs);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_rhs, xmm_sum);
|
||||
FMA_INT8_SSE(xmm_lhs, xmm_lhs, xmm_sum_norm1);
|
||||
FMA_INT8_SSE(xmm_rhs, xmm_rhs, xmm_sum_norm2);
|
||||
lhs += 16;
|
||||
rhs += 16;
|
||||
}
|
||||
}
|
||||
float result = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum));
|
||||
float norm1 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm1));
|
||||
float norm2 = static_cast<float>(HorizontalAdd_INT32_V128(xmm_sum_norm2));
|
||||
|
||||
switch (last - lhs) {
|
||||
case 15:
|
||||
FMA_INT8_GENERAL(lhs[14], rhs[14], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 14:
|
||||
FMA_INT8_GENERAL(lhs[13], rhs[13], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 13:
|
||||
FMA_INT8_GENERAL(lhs[12], rhs[12], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 12:
|
||||
FMA_INT8_GENERAL(lhs[11], rhs[11], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 11:
|
||||
FMA_INT8_GENERAL(lhs[10], rhs[10], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 10:
|
||||
FMA_INT8_GENERAL(lhs[9], rhs[9], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 9:
|
||||
FMA_INT8_GENERAL(lhs[8], rhs[8], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 8:
|
||||
FMA_INT8_GENERAL(lhs[7], rhs[7], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 7:
|
||||
FMA_INT8_GENERAL(lhs[6], rhs[6], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 6:
|
||||
FMA_INT8_GENERAL(lhs[5], rhs[5], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 5:
|
||||
FMA_INT8_GENERAL(lhs[4], rhs[4], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 4:
|
||||
FMA_INT8_GENERAL(lhs[3], rhs[3], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 3:
|
||||
FMA_INT8_GENERAL(lhs[2], rhs[2], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 2:
|
||||
FMA_INT8_GENERAL(lhs[1], rhs[1], result, norm1, norm2)
|
||||
/* FALLTHRU */
|
||||
case 1:
|
||||
FMA_INT8_GENERAL(lhs[0], rhs[0], result, norm1, norm2)
|
||||
}
|
||||
*sql = norm1;
|
||||
*sqr = norm2;
|
||||
return result;
|
||||
}
|
||||
#endif // __SSE4_1__
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -73,13 +73,17 @@ void Norm1Matrix<Float16, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP16_1_NEON(m, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP16_1_AVX512(m, dim, out, )
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP16_1_AVX512(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP16_1_AVX(m, dim, out, )
|
||||
#endif
|
||||
}
|
||||
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
@@ -26,18 +26,15 @@ namespace ailego {
|
||||
#define NORM_FP32_STEP_NEON SA_FP32_NEON
|
||||
|
||||
#if defined(__SSE__)
|
||||
static const __m128 ABS_MASK_FP32_SSE =
|
||||
_mm_castsi128_ps(_mm_set1_epi32(0x7fffffffu));
|
||||
#define ABS_MASK_FP32_SSE _mm_castsi128_ps(_mm_set1_epi32(0x7fffffffu))
|
||||
#endif // __SSE__
|
||||
|
||||
#if defined(__AVX__)
|
||||
static const __m256 ABS_MASK_FP32_AVX =
|
||||
_mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffffu));
|
||||
#define ABS_MASK_FP32_AVX _mm256_castsi256_ps(_mm256_set1_epi32(0x7fffffffu))
|
||||
#endif // __AVX__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
static const __m512 ABS_MASK_FP32_AVX512 =
|
||||
_mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffffu));
|
||||
#define ABS_MASK_FP32_AVX512 _mm512_castsi512_ps(_mm512_set1_epi32(0x7fffffffu))
|
||||
#endif // __AVX512F__
|
||||
|
||||
//! Calculate sum of absolute (GENERAL)
|
||||
@@ -64,15 +61,23 @@ void Norm1Matrix<float, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP32_1_NEON(m, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP32_1_AVX512(m, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
NORM_FP32_1_AVX(m, dim, out, )
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP32_1_AVX512(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
NORM_FP32_1_AVX(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP32_1_SSE(m, dim, out, )
|
||||
#endif
|
||||
}
|
||||
#endif // __SSE__ || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
@@ -58,9 +58,13 @@ void Norm2Matrix<Float16, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP16_1_NEON(m, dim, out, std::sqrt)
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP16_1_AVX512(m, dim, out, std::sqrt)
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP16_1_AVX512(m, dim, out, std::sqrt)
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP16_1_AVX(m, dim, out, std::sqrt)
|
||||
#endif
|
||||
}
|
||||
@@ -70,9 +74,13 @@ void SquaredNorm2Matrix<Float16, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP16_1_NEON(m, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP16_1_AVX512(m, dim, out, )
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP16_1_AVX512(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP16_1_AVX(m, dim, out, )
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include "norm2_matrix.h"
|
||||
#include "norm_matrix_fp32.i"
|
||||
|
||||
@@ -48,11 +49,19 @@ void Norm2Matrix<float, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP32_1_NEON(m, dim, out, std::sqrt)
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP32_1_AVX512(m, dim, out, std::sqrt)
|
||||
#elif defined(__AVX__)
|
||||
NORM_FP32_1_AVX(m, dim, out, std::sqrt)
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP32_1_AVX512(m, dim, out, std::sqrt)
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
NORM_FP32_1_AVX(m, dim, out, std::sqrt)
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP32_1_SSE(m, dim, out, std::sqrt)
|
||||
#endif
|
||||
}
|
||||
@@ -62,15 +71,23 @@ void SquaredNorm2Matrix<float, 1>::Compute(const ValueType *m, size_t dim,
|
||||
float *out) {
|
||||
#if defined(__ARM_NEON)
|
||||
NORM_FP32_1_NEON(m, dim, out, )
|
||||
#elif defined(__AVX512F__)
|
||||
NORM_FP32_1_AVX512(m, dim, out, )
|
||||
#elif defined(__AVX__)
|
||||
NORM_FP32_1_AVX(m, dim, out, )
|
||||
#else
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
NORM_FP32_1_AVX512(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
|
||||
NORM_FP32_1_AVX(m, dim, out, )
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
NORM_FP32_1_SSE(m, dim, out, )
|
||||
#endif
|
||||
}
|
||||
#endif // __SSE__ || (__ARM_NEON && __aarch64__)
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
@@ -19,10 +19,8 @@
|
||||
#include "cosine_distance_batch.h"
|
||||
#include "inner_product_distance_batch.h"
|
||||
|
||||
|
||||
namespace zvec::ailego {
|
||||
|
||||
|
||||
template <
|
||||
template <typename, size_t, size_t, typename = void> class DistanceType,
|
||||
typename ValueType, size_t BatchSize, size_t PrefetchStep, typename = void>
|
||||
|
||||
@@ -15,20 +15,28 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/math_batch/utils.h>
|
||||
#include <zvec/ailego/utility/type_helper.h>
|
||||
#include "inner_product_distance_batch_impl.h"
|
||||
#include "inner_product_distance_batch_impl_fp16.h"
|
||||
#include "inner_product_distance_batch_impl_int8.h"
|
||||
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
template <typename T, size_t BatchSize, size_t PrefetchStep, typename = void>
|
||||
struct InnerProductDistanceBatch;
|
||||
|
||||
template <typename ValueType, size_t BatchSize>
|
||||
static void compute_one_to_many_inner_product_fallback(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
for (size_t j = 0; j < BatchSize; ++j) {
|
||||
sums[j] = 0.0;
|
||||
InnerProductMatrix<ValueType, 1, 1>::Compute(ptrs[j], query, dim, sums + j);
|
||||
ailego_prefetch(&prefetch_ptrs[j]);
|
||||
}
|
||||
}
|
||||
|
||||
// Function template partial specialization is not allowed,
|
||||
// therefore the wrapper struct is required.
|
||||
template <typename T, size_t BatchSize>
|
||||
@@ -38,98 +46,14 @@ struct InnerProductDistanceBatchImpl {
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs,
|
||||
prefetch_ptrs, dim, sums);
|
||||
}
|
||||
static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() {
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t BatchSize>
|
||||
struct InnerProductDistanceBatchImpl<float, BatchSize> {
|
||||
using ValueType = float;
|
||||
static void compute_one_to_many(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_avx2_fp32<ValueType, BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() {
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t BatchSize>
|
||||
struct InnerProductDistanceBatchImpl<int8_t, BatchSize> {
|
||||
using ValueType = int8_t;
|
||||
static void compute_one_to_many(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
// #if defined(__AVX512BW__) // TODO: this version is problematic
|
||||
// return compute_one_to_many_avx512_int8<ValueType, BatchSize>(
|
||||
// query, ptrs, prefetch_ptrs, dim, sums);
|
||||
#if defined(__AVX512VNNI__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
return compute_one_to_many_avx512_vnni_int8<BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_avx2_int8<ValueType, BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() {
|
||||
#if defined(__AVX512VNNI__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
return compute_one_to_many_avx512_vnni_int8_query_preprocess;
|
||||
}
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t BatchSize>
|
||||
struct InnerProductDistanceBatchImpl<ailego::Float16, BatchSize> {
|
||||
using ValueType = ailego::Float16;
|
||||
static void compute_one_to_many(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
return compute_one_to_many_avx512fp16_fp16<ValueType, BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
return compute_one_to_many_avx512f_fp16<ValueType, BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_avx2_fp16<ValueType, BatchSize>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, size_t BatchSize, size_t PrefetchStep, typename>
|
||||
struct InnerProductDistanceBatch {
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
@@ -163,4 +87,56 @@ struct InnerProductDistanceBatch {
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<ailego::Float16, 1> {
|
||||
using ValueType = ailego::Float16;
|
||||
static void compute_one_to_many(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs, size_t dim,
|
||||
float *sums);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<float, 1> {
|
||||
using ValueType = float;
|
||||
static void compute_one_to_many(const float *query, const float **ptrs,
|
||||
std::array<const float *, 1> &prefetch_ptrs,
|
||||
size_t dim, float *sums);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<int8_t, 1> {
|
||||
using ValueType = int8_t;
|
||||
static void compute_one_to_many(const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs,
|
||||
size_t dim, float *sums);
|
||||
|
||||
static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc();
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<ailego::Float16, 12> {
|
||||
using ValueType = ailego::Float16;
|
||||
static void compute_one_to_many(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs, size_t dim,
|
||||
float *sums);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<float, 12> {
|
||||
using ValueType = float;
|
||||
static void compute_one_to_many(const float *query, const float **ptrs,
|
||||
std::array<const float *, 12> &prefetch_ptrs,
|
||||
size_t dim, float *sums);
|
||||
};
|
||||
|
||||
template <>
|
||||
struct InnerProductDistanceBatchImpl<int8_t, 12> {
|
||||
using ValueType = int8_t;
|
||||
static void compute_one_to_many(const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs,
|
||||
size_t dim, float *sums);
|
||||
};
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <ailego/internal/cpu_features.h>
|
||||
#include <ailego/math/inner_product_matrix.h>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/math_batch/utils.h>
|
||||
#include <zvec/ailego/utility/type_helper.h>
|
||||
#include "inner_product_distance_batch.h"
|
||||
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
#if defined(__AVX512VNNI__)
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess(
|
||||
void *query, size_t dim);
|
||||
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_1(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_12(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
void compute_one_to_many_inner_product_avx512fp16_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx512fp16_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
#endif //__AVX512FP16__
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void compute_one_to_many_inner_product_avx512f_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx512f_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
#endif //__AVX512F__
|
||||
|
||||
#if defined(__AVX2__)
|
||||
void compute_one_to_many_inner_product_avx2_fp32_1(
|
||||
const float *query, const float **ptrs,
|
||||
std::array<const float *, 1> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_int8_1(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp32_12(
|
||||
const float *query, const float **ptrs,
|
||||
std::array<const float *, 12> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results);
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_int8_12(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results);
|
||||
#endif
|
||||
|
||||
void InnerProductDistanceBatchImpl<float, 1>::compute_one_to_many(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, 1> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_fp32_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
void InnerProductDistanceBatchImpl<ailego::Float16, 1>::compute_one_to_many(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
return compute_one_to_many_inner_product_avx512fp16_fp16_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
return compute_one_to_many_inner_product_avx512f_fp16_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_fp16_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
void InnerProductDistanceBatchImpl<int8_t, 1>::compute_one_to_many(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
// #if defined(__AVX512BW__) // TODO: this version is problematic
|
||||
// return compute_one_to_many_avx512_int8<ValueType, BatchSize>(
|
||||
// query, ptrs, prefetch_ptrs, dim, sums);
|
||||
#if defined(__AVX512VNNI__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
return compute_one_to_many_inner_product_avx512_vnni_int8_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_int8_1(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
DistanceBatchQueryPreprocessFunc
|
||||
InnerProductDistanceBatchImpl<int8_t, 1>::GetQueryPreprocessFunc() {
|
||||
#if defined(__AVX512VNNI__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
return compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess;
|
||||
}
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void InnerProductDistanceBatchImpl<float, 12>::compute_one_to_many(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, 12> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_fp32_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
void InnerProductDistanceBatchImpl<ailego::Float16, 12>::compute_one_to_many(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
#if defined(__AVX512FP16__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
|
||||
return compute_one_to_many_inner_product_avx512fp16_fp16_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX512F__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
|
||||
return compute_one_to_many_inner_product_avx512f_fp16_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_fp16_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
void InnerProductDistanceBatchImpl<int8_t, 12>::compute_one_to_many(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
// #if defined(__AVX512BW__) // TODO: this version is problematic
|
||||
// return compute_one_to_many_avx512_int8<ValueType, BatchSize>(
|
||||
// query, ptrs, prefetch_ptrs, dim, sums);
|
||||
#if defined(__AVX512VNNI__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
|
||||
return compute_one_to_many_inner_product_avx512_vnni_int8_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
#if defined(__AVX2__)
|
||||
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
|
||||
return compute_one_to_many_inner_product_avx2_int8_12(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
return compute_one_to_many_inner_product_fallback(query, ptrs, prefetch_ptrs,
|
||||
dim, sums);
|
||||
}
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <array>
|
||||
#include <ailego/math/matrix_utility.i>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
|
||||
compute_one_to_many_inner_product_avx2_fp16(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m256, dp_batch> accs;
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_ps();
|
||||
}
|
||||
|
||||
size_t dim = 0;
|
||||
for (; dim + 16 <= dimensionality; dim += 16) {
|
||||
__m256i q =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(query + dim));
|
||||
|
||||
__m256 q1 = _mm256_cvtph_ps(_mm256_castsi256_si128(q));
|
||||
__m256 q2 = _mm256_cvtph_ps(_mm256_extractf128_si256(q, 1));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs_1;
|
||||
std::array<__m256, dp_batch> data_regs_2;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m256i m =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptrs[i] + dim));
|
||||
|
||||
data_regs_1[i] = _mm256_cvtph_ps(_mm256_castsi256_si128(m));
|
||||
data_regs_2[i] = _mm256_cvtph_ps(_mm256_extractf128_si256(m, 1));
|
||||
}
|
||||
|
||||
if (prefetch_ptrs[0]) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
ailego_prefetch(prefetch_ptrs[i] + dim);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_fmadd_ps(q1, data_regs_1[i], accs[i]);
|
||||
accs[i] = _mm256_fmadd_ps(q2, data_regs_2[i], accs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (dim + 8 <= dimensionality) {
|
||||
__m256 q = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(query + dim)));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(ptrs[i] + dim)));
|
||||
accs[i] = _mm256_fmadd_ps(q, data_regs[i], accs[i]);
|
||||
}
|
||||
|
||||
dim += 8;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] = HorizontalAdd_FP32_V256(accs[i]);
|
||||
}
|
||||
|
||||
for (; dim < dimensionality; ++dim) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] += (*(query + dim)) * (*(ptrs[i] + dim));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_fp16<ailego::Float16, 1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_fp16<ailego::Float16, 12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
+35
-73
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <ailego/math/matrix_utility.i>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
@@ -23,10 +21,9 @@
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
#if defined(__AVX512FP16__)
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
|
||||
compute_one_to_many_avx512fp16_fp16(
|
||||
compute_one_to_many_inner_product_avx512fp16_fp16(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
@@ -82,7 +79,7 @@ compute_one_to_many_avx512fp16_fp16(
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
|
||||
compute_one_to_many_avx512f_fp16(
|
||||
compute_one_to_many_inner_product_avx512f_fp16(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
@@ -122,7 +119,7 @@ compute_one_to_many_avx512f_fp16(
|
||||
}
|
||||
}
|
||||
|
||||
if (dim + 16 < dimensionality) {
|
||||
if (dim + 16 <= dimensionality) {
|
||||
__m512 q = _mm512_cvtph_ps(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(query + dim)));
|
||||
|
||||
@@ -143,7 +140,7 @@ compute_one_to_many_avx512f_fp16(
|
||||
_mm256_castpd_ps(_mm512_extractf64x4_pd(_mm512_castps_pd(accs[i]), 1)));
|
||||
}
|
||||
|
||||
if (dim + 8 < dimensionality) {
|
||||
if (dim + 8 <= dimensionality) {
|
||||
__m256 q = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(query + dim)));
|
||||
|
||||
@@ -166,78 +163,43 @@ compute_one_to_many_avx512f_fp16(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
|
||||
compute_one_to_many_avx2_fp16(
|
||||
#if defined(__AVX512FP16__)
|
||||
void compute_one_to_many_inner_product_avx512fp16_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m256, dp_batch> accs;
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_ps();
|
||||
}
|
||||
|
||||
size_t dim = 0;
|
||||
for (; dim + 16 <= dimensionality; dim += 16) {
|
||||
__m256i q =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(query + dim));
|
||||
|
||||
__m256 q1 = _mm256_cvtph_ps(_mm256_castsi256_si128(q));
|
||||
__m256 q2 = _mm256_cvtph_ps(_mm256_extractf128_si256(q, 1));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs_1;
|
||||
std::array<__m256, dp_batch> data_regs_2;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m256i m =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptrs[i] + dim));
|
||||
|
||||
data_regs_1[i] = _mm256_cvtph_ps(_mm256_castsi256_si128(m));
|
||||
data_regs_2[i] = _mm256_cvtph_ps(_mm256_extractf128_si256(m, 1));
|
||||
}
|
||||
|
||||
if (prefetch_ptrs[0]) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
ailego_prefetch(prefetch_ptrs[i] + dim);
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_fmadd_ps(q1, data_regs_1[i], accs[i]);
|
||||
accs[i] = _mm256_fmadd_ps(q2, data_regs_2[i], accs[i]);
|
||||
}
|
||||
}
|
||||
|
||||
if (dim + 8 < dimensionality) {
|
||||
__m256 q = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(query + dim)));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(ptrs[i] + dim)));
|
||||
accs[i] = _mm256_fmadd_ps(q, data_regs[i], accs[i]);
|
||||
}
|
||||
|
||||
dim += 8;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] = HorizontalAdd_FP32_V256(accs[i]);
|
||||
}
|
||||
|
||||
for (; dim < dimensionality; ++dim) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] += (*(query + dim)) * (*(ptrs[i] + dim));
|
||||
}
|
||||
}
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512fp16_fp16<ailego::Float16, 1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx512fp16_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512fp16_fp16<ailego::Float16, 12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(__AVX512F__)
|
||||
void compute_one_to_many_inner_product_avx512f_fp16_1(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 1> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512f_fp16<ailego::Float16, 1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx512f_fp16_12(
|
||||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, 12> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512f_fp16<ailego::Float16, 12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
+16
-16
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <ailego/math/inner_product_matrix.h>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
@@ -22,18 +20,6 @@
|
||||
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
template <typename ValueType, size_t BatchSize>
|
||||
static void compute_one_to_many_fallback(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, BatchSize> &prefetch_ptrs, size_t dim,
|
||||
float *sums) {
|
||||
for (size_t j = 0; j < BatchSize; ++j) {
|
||||
sums[j] = 0.0;
|
||||
InnerProductMatrix<ValueType, 1, 1>::Compute(ptrs[j], query, dim, sums + j);
|
||||
ailego_prefetch(&prefetch_ptrs[j]);
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
inline float sum4(__m128 v) {
|
||||
@@ -49,7 +35,7 @@ inline __m128 sum_top_bottom_avx(__m256 v) {
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, float>, void>
|
||||
compute_one_to_many_avx2_fp32(
|
||||
compute_one_to_many_inner_product_avx2_fp32(
|
||||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
@@ -123,7 +109,21 @@ compute_one_to_many_avx2_fp32(
|
||||
results[i] = -res[i];
|
||||
}
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp32_1(
|
||||
const float *query, const float **ptrs,
|
||||
std::array<const float *, 1> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_fp32<float, 1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_fp32_12(
|
||||
const float *query, const float **ptrs,
|
||||
std::array<const float *, 12> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_fp32<float, 12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, int8_t>, void>
|
||||
compute_one_to_many_inner_product_avx2_int8(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, dp_batch> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results) {
|
||||
std::vector<__m256i> accs(dp_batch);
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_si256();
|
||||
}
|
||||
size_t dim = 0;
|
||||
for (; dim + 32 <= dimensionality; dim += 32) {
|
||||
__m256i q = _mm256_loadu_si256((const __m256i *)(query + dim));
|
||||
std::vector<__m256i> data_regs(dp_batch);
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_loadu_si256((const __m256i *)(ptrs[i] + dim));
|
||||
}
|
||||
if (prefetch_ptrs[0]) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
ailego_prefetch(prefetch_ptrs[i] + dim);
|
||||
}
|
||||
}
|
||||
__m256i q_lo = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(q));
|
||||
__m256i q_hi = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(q, 1));
|
||||
__m256i data_lo[dp_batch];
|
||||
__m256i data_hi[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_lo[i] = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(data_regs[i]));
|
||||
data_hi[i] =
|
||||
_mm256_cvtepi8_epi16(_mm256_extracti128_si256(data_regs[i], 1));
|
||||
}
|
||||
__m256i prod_lo[dp_batch];
|
||||
__m256i prod_hi[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
prod_lo[i] = _mm256_madd_epi16(q_lo, data_lo[i]);
|
||||
prod_hi[i] = _mm256_madd_epi16(q_hi, data_hi[i]);
|
||||
}
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] =
|
||||
_mm256_add_epi32(accs[i], _mm256_add_epi32(prod_lo[i], prod_hi[i]));
|
||||
}
|
||||
}
|
||||
std::array<int, dp_batch> temp_results;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m128i lo = _mm256_castsi256_si128(accs[i]);
|
||||
__m128i hi = _mm256_extracti128_si256(accs[i], 1);
|
||||
__m128i sum128 = _mm_add_epi32(lo, hi);
|
||||
sum128 = _mm_hadd_epi32(sum128, sum128);
|
||||
sum128 = _mm_hadd_epi32(sum128, sum128);
|
||||
temp_results[i] = _mm_cvtsi128_si32(sum128);
|
||||
}
|
||||
for (; dim < dimensionality; ++dim) {
|
||||
int8_t q = query[dim];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
temp_results[i] += q * static_cast<int>(ptrs[i][dim]);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] = static_cast<float>(temp_results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_int8_1(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_int8<int8_t, 1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx2_int8_12(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx2_int8<int8_t, 12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
+15
-69
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
@@ -23,8 +21,8 @@ namespace zvec::ailego::DistanceBatch {
|
||||
|
||||
#if defined(__AVX512VNNI__)
|
||||
|
||||
static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query,
|
||||
size_t dim) {
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_query_preprocess(
|
||||
void *query, size_t dim) {
|
||||
const int8_t *input = reinterpret_cast<const int8_t *>(query);
|
||||
uint8_t *output = reinterpret_cast<uint8_t *>(query);
|
||||
|
||||
@@ -48,10 +46,9 @@ static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// query is unsigned
|
||||
template <size_t dp_batch>
|
||||
static void compute_one_to_many_avx512_vnni_int8(
|
||||
static void compute_one_to_many_inner_product_avx512_vnni_int8(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, dp_batch> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results) {
|
||||
@@ -153,73 +150,22 @@ static void compute_one_to_many_avx512_vnni_int8(
|
||||
// results[i] = static_cast<float>(temp_results[i]);
|
||||
// }
|
||||
// }
|
||||
#endif
|
||||
|
||||
#if defined(__AVX2__)
|
||||
|
||||
template <typename ValueType, size_t dp_batch>
|
||||
static std::enable_if_t<std::is_same_v<ValueType, int8_t>, void>
|
||||
compute_one_to_many_avx2_int8(
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_1(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, dp_batch> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results) {
|
||||
std::array<__m256i, dp_batch> accs;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_si256();
|
||||
}
|
||||
size_t dim = 0;
|
||||
for (; dim + 32 <= dimensionality; dim += 32) {
|
||||
__m256i q = _mm256_loadu_si256((const __m256i *)(query + dim));
|
||||
std::array<__m256i, dp_batch> data_regs;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_loadu_si256((const __m256i *)(ptrs[i] + dim));
|
||||
}
|
||||
if (prefetch_ptrs[0]) {
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
ailego_prefetch(prefetch_ptrs[i] + dim);
|
||||
}
|
||||
}
|
||||
__m256i q_lo = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(q));
|
||||
__m256i q_hi = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(q, 1));
|
||||
std::array<__m256i, dp_batch> data_lo;
|
||||
std::array<__m256i, dp_batch> data_hi;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_lo[i] = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(data_regs[i]));
|
||||
data_hi[i] =
|
||||
_mm256_cvtepi8_epi16(_mm256_extracti128_si256(data_regs[i], 1));
|
||||
}
|
||||
std::array<__m256i, dp_batch> prod_lo;
|
||||
std::array<__m256i, dp_batch> prod_hi;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
prod_lo[i] = _mm256_madd_epi16(q_lo, data_lo[i]);
|
||||
prod_hi[i] = _mm256_madd_epi16(q_hi, data_hi[i]);
|
||||
}
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] =
|
||||
_mm256_add_epi32(accs[i], _mm256_add_epi32(prod_lo[i], prod_hi[i]));
|
||||
}
|
||||
}
|
||||
std::array<int, dp_batch> temp_results;
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m128i lo = _mm256_castsi256_si128(accs[i]);
|
||||
__m128i hi = _mm256_extracti128_si256(accs[i], 1);
|
||||
__m128i sum128 = _mm_add_epi32(lo, hi);
|
||||
sum128 = _mm_hadd_epi32(sum128, sum128);
|
||||
sum128 = _mm_hadd_epi32(sum128, sum128);
|
||||
temp_results[i] = _mm_cvtsi128_si32(sum128);
|
||||
}
|
||||
for (; dim < dimensionality; ++dim) {
|
||||
int8_t q = query[dim];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
temp_results[i] += q * static_cast<int>(ptrs[i][dim]);
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
results[i] = static_cast<float>(temp_results[i]);
|
||||
}
|
||||
std::array<const int8_t *, 1> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512_vnni_int8<1>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
void compute_one_to_many_inner_product_avx512_vnni_int8_12(
|
||||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, 12> &prefetch_ptrs, size_t dim, float *sums) {
|
||||
return compute_one_to_many_inner_product_avx512_vnni_int8<12>(
|
||||
query, ptrs, prefetch_ptrs, dim, sums);
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
} // namespace zvec::ailego::DistanceBatch
|
||||
Reference in New Issue
Block a user