diff --git a/src/ailego/internal/cpu_features.cc b/src/ailego/internal/cpu_features.cc index 7b16e3a..9b8ea82 100644 --- a/src/ailego/internal/cpu_features.cc +++ b/src/ailego/internal/cpu_features.cc @@ -416,6 +416,7 @@ const char *CpuFeatures::Intrinsics(void) { ; } +CpuFeatures::StaticFlags CpuFeatures::static_flags_; } // namespace internal } // namespace ailego } // namespace zvec \ No newline at end of file diff --git a/src/ailego/internal/cpu_features.h b/src/ailego/internal/cpu_features.h index eaa39a6..b789a8d 100644 --- a/src/ailego/internal/cpu_features.h +++ b/src/ailego/internal/cpu_features.h @@ -201,6 +201,169 @@ class CpuFeatures { //! Static Members static CpuFlags flags_; + + public: + struct StaticFlags { + //! 16-bit FP conversions + bool F16C = CpuFeatures::F16C(); + + //! Multimedia Extensions + bool MMX = CpuFeatures::MMX(); + + //! Streaming SIMD Extensions + bool SSE = CpuFeatures::SSE(); + + //! Streaming SIMD Extensions 2 + bool SSE2 = CpuFeatures::SSE2(); + + //! Streaming SIMD Extensions 3 + bool SSE3 = CpuFeatures::SSE3(); + + //! Supplemental Streaming SIMD Extensions 3 + bool SSSE3 = CpuFeatures::SSSE3(); + + //! Streaming SIMD Extensions 4.1 + bool SSE4_1 = CpuFeatures::SSE4_1(); + + //! Streaming SIMD Extensions 4.2 + bool SSE4_2 = CpuFeatures::SSE4_2(); + + //! Advanced Vector Extensions + bool AVX = CpuFeatures::AVX(); + + //! Advanced Vector Extensions 2 + bool AVX2 = CpuFeatures::AVX2(); + + //! AVX-512 Foundation + bool AVX512F = CpuFeatures::AVX512F(); + + //! AVX-512 DQ (Double/Quad granular) Instructions + bool AVX512DQ = CpuFeatures::AVX512DQ(); + + //! AVX-512 Prefetch + bool AVX512PF = CpuFeatures::AVX512PF(); + + //! AVX-512 Exponential and Reciprocal + bool AVX512ER = CpuFeatures::AVX512ER(); + + //! AVX-512 Conflict Detection + bool AVX512CD = CpuFeatures::AVX512CD(); + + //! AVX-512 BW (Byte/Word granular) Instructions + bool AVX512BW = CpuFeatures::AVX512BW(); + + //! AVX-512 VL (128/256 Vector Length) Extensions + bool AVX512VL = CpuFeatures::AVX512VL(); + + //! AVX-512 Integer Fused Multiply-Add instructions + bool AVX512_IFMA = CpuFeatures::AVX512_IFMA(); + + //! AVX512 Vector Bit Manipulation instructions + bool AVX512_VBMI = CpuFeatures::AVX512_VBMI(); + + //! Additional AVX512 Vector Bit Manipulation Instructions + bool AVX512_VBMI2 = CpuFeatures::AVX512_VBMI2(); + + //! Vector Neural Network Instructions + bool AVX512_VNNI = CpuFeatures::AVX512_VNNI(); + + //! Support for VPOPCNT[B,W] and VPSHUF-BITQMB instructions + bool AVX512_BITALG = CpuFeatures::AVX512_BITALG(); + + //! POPCNT for vectors of DW/QW + bool AVX512_VPOPCNTDQ = CpuFeatures::AVX512_VPOPCNTDQ(); + + //! AVX-512 Neural Network Instructions + bool AVX512_4VNNIW = CpuFeatures::AVX512_4VNNIW(); + + //! AVX-512 Multiply Accumulation Single precision + bool AVX512_4FMAPS = CpuFeatures::AVX512_4FMAPS(); + + //! AVX-512 FP16 instructions + bool AVX512_FP16 = CpuFeatures::AVX512_FP16(); + + //! CMPXCHG8 instruction + bool CX8 = CpuFeatures::CX8(); + + //! CMPXCHG16B instruction + bool CX16 = CpuFeatures::CX16(); + + //! PCLMULQDQ instruction + bool PCLMULQDQ = CpuFeatures::PCLMULQDQ(); + + //! Carry-Less Multiplication Double Quadword + bool VPCLMULQDQ = CpuFeatures::VPCLMULQDQ(); + + //! CMOV instructions (plus FCMOVcc, FCOMI with FPU) + bool CMOV = CpuFeatures::CMOV(); + + //! MOVBE instruction + bool MOVBE = CpuFeatures::MOVBE(); + + //! Enhanced REP MOVSB/STOSB instructions + bool ERMS = CpuFeatures::ERMS(); + + //! POPCNT instruction + bool POPCNT = CpuFeatures::POPCNT(); + + //! XSAVE/XRSTOR/XSETBV/XGETBV instructions + bool XSAVE = CpuFeatures::XSAVE(); + + //! Fused multiply-add + bool FMA = CpuFeatures::FMA(); + + //! ADCX and ADOX instructions + bool ADX = CpuFeatures::ADX(); + + //! Galois Field New Instructions + bool GFNI = CpuFeatures::GFNI(); + + //! AES instructions + bool AES = CpuFeatures::AES(); + + //! Vector AES + bool VAES = CpuFeatures::VAES(); + + //! RDSEED instruction + bool RDSEED = CpuFeatures::RDSEED(); + + //! RDRAND instruction + bool RDRAND = CpuFeatures::RDRAND(); + + //! SHA1/SHA256 Instruction Extensions + bool SHA = CpuFeatures::SHA(); + + //! 1st group bit manipulation extensions + bool BMI1 = CpuFeatures::BMI1(); + + //! 2nd group bit manipulation extensions + bool BMI2 = CpuFeatures::BMI2(); + + //! CLFLUSH instruction + bool CLFLUSH = CpuFeatures::CLFLUSH(); + + //! CLFLUSHOPT instruction + bool CLFLUSHOPT = CpuFeatures::CLFLUSHOPT(); + + //! CLWB instruction + bool CLWB = CpuFeatures::CLWB(); + + //! RDPID instruction + bool RDPID = CpuFeatures::RDPID(); + + //! Onboard FPU + bool FPU = CpuFeatures::FPU(); + + //! Hyper-Threading + bool HT = CpuFeatures::HT(); + + //! Hardware virtualization + bool VMX = CpuFeatures::VMX(); + + // !Running on a hypervisor + bool HYPERVISOR = CpuFeatures::HYPERVISOR(); + }; + static StaticFlags static_flags_; }; } // namespace internal diff --git a/src/ailego/math/distance_matrix_fp16.i b/src/ailego/math/distance_matrix_fp16.i index 8cd558e..ac8b434 100644 --- a/src/ailego/math/distance_matrix_fp16.i +++ b/src/ailego/math/distance_matrix_fp16.i @@ -14,7 +14,7 @@ #include #include "matrix_define.i" - +#include #if !defined(__AVX__) #define _mm_broadcast_si32(a) _mm_castps_si128(_mm_load1_ps((const float *)(a))) #else @@ -95,13 +95,13 @@ } \ case 1: { \ __m256 ymm_lhs = _mm256_cvtph_ps( \ - _mm_set_epi16((short)(_MASK), (short)(_MASK), (short)(_MASK), \ + _mm_set_epi16(*((const short *)(lhs)), (short)(_MASK), \ (short)(_MASK), (short)(_MASK), (short)(_MASK), \ - (short)(_MASK), *((const short *)(lhs)))); \ + (short)(_MASK), (short)(_MASK), (short)(_MASK))); \ __m256 ymm_rhs = _mm256_cvtph_ps( \ - _mm_set_epi16((short)(_MASK), (short)(_MASK), (short)(_MASK), \ + _mm_set_epi16(*((const short *)(rhs)), (short)(_MASK), \ (short)(_MASK), (short)(_MASK), (short)(_MASK), \ - (short)(_MASK), *((const short *)(rhs)))); \ + (short)(_MASK), (short)(_MASK), (short)(_MASK))); \ _PROC(ymm_lhs, ymm_rhs, _RES##_0_0) \ break; \ } \ diff --git a/src/ailego/math/euclidean_distance_matrix.h b/src/ailego/math/euclidean_distance_matrix.h index cb0cf2c..a70d3bb 100644 --- a/src/ailego/math/euclidean_distance_matrix.h +++ b/src/ailego/math/euclidean_distance_matrix.h @@ -178,14 +178,15 @@ struct SquaredEuclideanDistanceMatrix< protected: //! Calculate the squared difference static inline float SquaredDifference(uint32_t lhs, uint32_t rhs) { - return static_cast(MathHelper::SquaredDifference( - (int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 24), (int8_t)(rhs >> 24))); + volatile int32_t sum = MathHelper::SquaredDifference( + (int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 24), (int8_t)(rhs >> 24)); + return static_cast(sum); } }; @@ -228,14 +229,15 @@ struct SquaredEuclideanDistanceMatrix(MathHelper::SquaredDifference( - (int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) + - MathHelper::SquaredDifference( - (int8_t)(lhs >> 24), (int8_t)(rhs >> 24))); + volatile int32_t sum = MathHelper::SquaredDifference( + (int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) + + MathHelper::SquaredDifference( + (int8_t)(lhs >> 24), (int8_t)(rhs >> 24)); + return static_cast(sum); } }; diff --git a/src/ailego/math/euclidean_distance_matrix_fp16.cc b/src/ailego/math/euclidean_distance_matrix_fp16.cc index 366642a..ca24561 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp16.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp16.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "distance_matrix_accum_fp16.i" #include "euclidean_distance_matrix.h" @@ -131,6 +132,7 @@ static inline float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs, } #endif + //! Compute the distance between matrix and query (FP16, M=1, N=1) void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, @@ -138,13 +140,21 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, float *out) { #if defined(__ARM_NEON) ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, ) -#elif defined(__AVX512FP16__) - *out = SquaredEuclideanDistanceAVX512FP16(m, q, dim); -#elif defined(__AVX512F__) - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) #else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = SquaredEuclideanDistanceAVX512FP16(m, q, dim); + return; + } +#endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) + return; + } +#endif ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, ) -#endif // __AVX512F__ +#endif //__ARM_NEON } //! Compute the distance between matrix and query (FP16, M=1, N=1) @@ -153,13 +163,21 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, std::sqrt) -#elif defined(__AVX512FP16__) - *out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim)); -#elif defined(__AVX512F__) - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt) #else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim)); + return; + } +#endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt) + return; + } +#endif ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, std::sqrt) -#endif // __AVX512F__ +#endif //__ARM_NEON } #if !defined(__ARM_NEON) @@ -241,10 +259,13 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X1_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X1_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X1_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + + ACCUM_FP16_16X1_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=16, N=2) @@ -253,10 +274,13 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X2_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X2_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X2_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + + ACCUM_FP16_16X2_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=16, N=4) @@ -265,10 +289,12 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X4_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X4_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X4_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X4_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=16, N=8) @@ -277,20 +303,24 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X8_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X8_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X8_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X8_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=16, N=16) void SquaredEuclideanDistanceMatrix::Compute( const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X16_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_16X16_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X16_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X16_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=1) @@ -299,10 +329,12 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X1_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X1_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X1_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X1_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=2) @@ -311,10 +343,12 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X2_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X2_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X2_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X2_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=4) @@ -323,10 +357,12 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X4_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X4_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X4_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X4_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=8) @@ -335,30 +371,36 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X8_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X8_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X8_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X8_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=16) void SquaredEuclideanDistanceMatrix::Compute( const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X16_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X16_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X16_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X16_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=32, N=32) void SquaredEuclideanDistanceMatrix::Compute( const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X32_AVX512(m, q, dim, out, ) -#else - ACCUM_FP16_32X32_AVX(m, q, dim, out, ) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X32_AVX512(m, q, dim, out, ) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X32_AVX(m, q, dim, out, ) } //! Compute the distance between matrix and query (FP16, M=2, N=1) @@ -429,10 +471,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=16, N=2) @@ -440,10 +484,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=16, N=4) @@ -451,10 +497,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=16, N=8) @@ -462,10 +510,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=16, N=16) @@ -473,10 +523,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=1) @@ -484,10 +536,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=2) @@ -495,10 +549,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=4) @@ -506,10 +562,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=8) @@ -517,10 +575,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=16) @@ -528,10 +588,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps) } //! Compute the distance between matrix and query (FP16, M=32, N=32) @@ -539,10 +601,12 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, const ValueType *q, size_t dim, float *out) { #if defined(__AVX512F__) - ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps) -#else - ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps) + return; + } #endif // __AVX512F__ + ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps) } #endif // !__ARM_NEON #endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__) diff --git a/src/ailego/math/euclidean_distance_matrix_fp32.cc b/src/ailego/math/euclidean_distance_matrix_fp32.cc index 9b49be3..7a02473 100644 --- a/src/ailego/math/euclidean_distance_matrix_fp32.cc +++ b/src/ailego/math/euclidean_distance_matrix_fp32.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "distance_matrix_accum_fp32.i" #include "euclidean_distance_matrix.h" @@ -291,15 +292,19 @@ void SquaredEuclideanDistanceMatrix::Compute(const ValueType *m, *out = SquaredEuclideanDistanceNEON(m, q, dim); #else #if defined(__AVX512F__) - if (dim > 15) { - *out = SquaredEuclideanDistanceAVX512(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = SquaredEuclideanDistanceAVX512(m, q, dim); + return; + } } #endif // __AVX512F__ #if defined(__AVX__) - if (dim > 7) { - *out = SquaredEuclideanDistanceAVX(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = SquaredEuclideanDistanceAVX(m, q, dim); + return; + } } #endif // __AVX__ *out = SquaredEuclideanDistanceSSE(m, q, dim); @@ -618,15 +623,20 @@ void EuclideanDistanceMatrix::Compute(const ValueType *m, *out = std::sqrt(SquaredEuclideanDistanceNEON(m, q, dim)); #else #if defined(__AVX512F__) - if (dim > 15) { - *out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim)); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim)); + return; + } } #endif // __AVX512F__ + #if defined(__AVX__) - if (dim > 7) { - *out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim)); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim)); + return; + } } #endif // __AVX__ *out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim)); diff --git a/src/ailego/math/inner_product_matrix.h b/src/ailego/math/inner_product_matrix.h index 5bd7e0a..71ac75c 100644 --- a/src/ailego/math/inner_product_matrix.h +++ b/src/ailego/math/inner_product_matrix.h @@ -181,10 +181,12 @@ struct InnerProductMatrix((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + - (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + - (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + - (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + + (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + + (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + + (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + + return static_cast(sum); } }; @@ -226,10 +228,12 @@ struct InnerProductMatrix= 2>::type> { protected: //! Calculate Fused-Multiply-Add static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) { - return static_cast((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + - (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + - (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + - (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + + (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + + (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + + (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + + return static_cast(sum); } }; @@ -524,10 +528,12 @@ struct MinusInnerProductMatrix< protected: //! Calculate Fused-Multiply-Add static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) { - return static_cast((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + - (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + - (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + - (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + + (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + + (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + + (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + + return static_cast(sum); } }; @@ -570,10 +576,12 @@ struct MinusInnerProductMatrix((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + - (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + - (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + - (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) + + (int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) + + (int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) + + (int8_t)(lhs >> 24) * (int8_t)(rhs >> 24)); + + return static_cast(sum); } }; diff --git a/src/ailego/math/inner_product_matrix_fp16.cc b/src/ailego/math/inner_product_matrix_fp16.cc index 6e2e8a6..682cc91 100644 --- a/src/ailego/math/inner_product_matrix_fp16.cc +++ b/src/ailego/math/inner_product_matrix_fp16.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "distance_matrix_accum_fp16.i" #include "inner_product_matrix.h" @@ -144,13 +145,21 @@ void InnerProductMatrix::Compute(const ValueType *m, float *out) { #if defined(__ARM_NEON) ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, ) -#elif defined(__AVX512FP16__) - *out = InnerProductAVX512FP16(m, q, dim); -#elif defined(__AVX512F__) - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) #else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = InnerProductAVX512FP16(m, q, dim); + return; + } +#endif //__AVX512FP16__ +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, ) + return; + } +#endif //__AVX512F__ ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, ) -#endif +#endif //__ARM_NEON } //! Compute the distance between matrix and query (FP16, M=1, N=1) @@ -159,13 +168,21 @@ void MinusInnerProductMatrix::Compute(const ValueType *m, size_t dim, float *out) { #if defined(__ARM_NEON) ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) -#elif defined(__AVX512FP16__) - *out = -InnerProductAVX512FP16(m, q, dim); -#elif defined(__AVX512F__) - ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) #else +#if defined(__AVX512FP16__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) { + *out = -InnerProductAVX512FP16(m, q, dim); + return; + } +#endif //__AVX512FP16__ +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) + return; + } +#endif //__AVX512F__ ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL) -#endif +#endif //__ARM_NEON } #if !defined(__ARM_NEON) diff --git a/src/ailego/math/inner_product_matrix_fp32.cc b/src/ailego/math/inner_product_matrix_fp32.cc index a93ba9b..78e260d 100644 --- a/src/ailego/math/inner_product_matrix_fp32.cc +++ b/src/ailego/math/inner_product_matrix_fp32.cc @@ -12,30 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "distance_matrix_accum_fp32.i" #include "inner_product_matrix.h" -#if DEBUG_PRINT -#include - -static void inline print_data16(__m128i *data) { - uint16_t buffer[16]; - - memcpy(buffer, data, sizeof(buffer)); - - // std::cout << "result equals: " << std::endl; - - for (int i = 0; i < 8; i++) { - uint64_t value = buffer[i]; - if (i < 8) { - std::cout << value << ", "; - } else { - std::cout << value << std::endl; - } - } -} -#endif - namespace zvec { namespace ailego { @@ -585,15 +565,19 @@ void InnerProductMatrix::Compute(const ValueType *m, *out = InnerProductNEON(m, q, dim); #else #if defined(__AVX512F__) - if (dim > 15) { - *out = InnerProductAVX512(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = InnerProductAVX512(m, q, dim); + return; + } } #endif // __AVX512F__ #if defined(__AVX__) - if (dim > 7) { - *out = InnerProductAVX(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = InnerProductAVX(m, q, dim); + return; + } } #endif // __AVX__ *out = InnerProductSSE(m, q, dim); @@ -890,15 +874,19 @@ void MinusInnerProductMatrix::Compute(const ValueType *m, *out = -InnerProductNEON(m, q, dim); #else #if defined(__AVX512F__) - if (dim > 15) { - *out = -InnerProductAVX512(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + if (dim > 15) { + *out = -InnerProductAVX512(m, q, dim); + return; + } } #endif // __AVX512F__ #if defined(__AVX__) - if (dim > 7) { - *out = -InnerProductAVX(m, q, dim); - return; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + if (dim > 7) { + *out = -InnerProductAVX(m, q, dim); + return; + } } #endif // __AVX__ *out = -InnerProductSSE(m, q, dim); diff --git a/src/ailego/math/inner_product_matrix_int4.cc b/src/ailego/math/inner_product_matrix_int4.cc index 2ae5eb4..87a82e8 100644 --- a/src/ailego/math/inner_product_matrix_int4.cc +++ b/src/ailego/math/inner_product_matrix_int4.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include "distance_matrix_accum_int4.i" #include "inner_product_matrix.h" diff --git a/src/ailego/math_batch/cosine_distance_batch.h b/src/ailego/math_batch/cosine_distance_batch.h index 345a879..d0185d3 100644 --- a/src/ailego/math_batch/cosine_distance_batch.h +++ b/src/ailego/math_batch/cosine_distance_batch.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include diff --git a/src/ailego/math_batch/inner_product_distance_batch.h b/src/ailego/math_batch/inner_product_distance_batch.h index c8c5217..5373e7d 100644 --- a/src/ailego/math_batch/inner_product_distance_batch.h +++ b/src/ailego/math_batch/inner_product_distance_batch.h @@ -15,6 +15,7 @@ #pragma once #include +#include #include #include #include @@ -75,19 +76,25 @@ struct InnerProductDistanceBatchImpl { // return compute_one_to_many_avx512_int8( // query, ptrs, prefetch_ptrs, dim, sums); #if defined(__AVX512VNNI__) - return compute_one_to_many_avx512_vnni_int8( - query, ptrs, prefetch_ptrs, dim, sums); -#elif defined(__AVX2__) - return compute_one_to_many_avx2_int8( - query, ptrs, prefetch_ptrs, dim, sums); -#else - return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + return compute_one_to_many_avx512_vnni_int8( + query, ptrs, prefetch_ptrs, dim, sums); + } #endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_avx2_int8( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); } static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() { #if defined(__AVX512VNNI__) - return compute_one_to_many_avx512_vnni_int8_query_preprocess; + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) { + return compute_one_to_many_avx512_vnni_int8_query_preprocess; + } #endif return nullptr; } @@ -107,11 +114,24 @@ struct InnerProductDistanceBatchImpl { return compute_one_to_many_avx512f_fp16( query, ptrs, prefetch_ptrs, dim, sums); #elif defined(__AVX2__) - return compute_one_to_many_avx_fp16( + return compute_one_to_many_avx2_fp16( query, ptrs, prefetch_ptrs, dim, sums); #else return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); #endif +#if defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return compute_one_to_many_avx512f_fp16( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif +#if defined(__AVX2__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) { + return compute_one_to_many_avx2_fp16( + query, ptrs, prefetch_ptrs, dim, sums); + } +#endif + return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums); } }; @@ -148,4 +168,4 @@ struct InnerProductDistanceBatch { } }; -} // namespace zvec::ailego::DistanceBatch \ No newline at end of file +} // namespace zvec::ailego::DistanceBatch diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h index 64bd1ae..7f4f72e 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h @@ -76,7 +76,9 @@ compute_one_to_many_avx512fp16_fp16( } } -#elif defined(__AVX512F__) +#endif + +#if defined(__AVX512F__) template static std::enable_if_t, void> @@ -164,12 +166,13 @@ compute_one_to_many_avx512f_fp16( } } } +#endif -#elif defined(__AVX2__) +#if defined(__AVX2__) template static std::enable_if_t, void> -compute_one_to_many_av2_fp16( +compute_one_to_many_avx2_fp16( const ailego::Float16 *query, const ailego::Float16 **ptrs, std::array &prefetch_ptrs, size_t dimensionality, float *results) { diff --git a/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h b/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h index e37388f..2f5be38 100644 --- a/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h +++ b/src/ailego/math_batch/inner_product_distance_batch_impl_int8.h @@ -21,7 +21,6 @@ namespace zvec::ailego::DistanceBatch { - #if defined(__AVX512VNNI__) static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query, @@ -154,8 +153,9 @@ static void compute_one_to_many_avx512_vnni_int8( // results[i] = static_cast(temp_results[i]); // } // } +#endif -#elif defined(__AVX2__) +#if defined(__AVX2__) template static std::enable_if_t, void> diff --git a/src/ailego/utility/float_helper.cc b/src/ailego/utility/float_helper.cc index 6c00793..164384a 100644 --- a/src/ailego/utility/float_helper.cc +++ b/src/ailego/utility/float_helper.cc @@ -13,579 +13,14 @@ // limitations under the License. #include "float_helper.h" +#include -#if defined(__F16C__) && defined(__AVX__) -#define float16(x) _cvtss_sh((x), _MM_FROUND_NO_EXC) -#define float32(x) _cvtsh_ss(x) -#endif // __F16C__ && __AVX__ +// #if defined(__F16C__) && defined(__AVX__) +// #define float16(x) _cvtss_sh((x), _MM_FROUND_NO_EXC) +// #define float32(x) _cvtsh_ss(x) +// #endif // __F16C__ && __AVX__ -#if defined(__F16C__) && defined(__AVX512F__) -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float *out) { - const uint16_t *last = arr + size; - const uint16_t *last_aligned = arr + ((size >> 5) << 5); - - if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0x3f) == 0) { - for (; arr != last_aligned; arr += 32, out += 32) { - _mm512_store_ps(out + 0, - _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 0)))); - _mm512_store_ps( - out + 16, _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 16)))); - } - - if (last >= last_aligned + 16) { - _mm512_store_ps(out, _mm512_cvtph_ps(_mm256_load_si256((__m256i *)arr))); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm256_store_ps(out, _mm256_cvtph_ps(_mm_load_si128((__m128i *)arr))); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 32, out += 32) { - _mm512_storeu_ps( - out + 0, _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 0)))); - _mm512_storeu_ps( - out + 16, _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 16)))); - } - - if (last >= last_aligned + 16) { - _mm512_storeu_ps(out, - _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)arr))); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm256_storeu_ps(out, _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr))); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtsh_ss(arr[6]); - /* FALLTHRU */ - case 6: - out[5] = _cvtsh_ss(arr[5]); - /* FALLTHRU */ - case 5: - out[4] = _cvtsh_ss(arr[4]); - /* FALLTHRU */ - case 4: - out[3] = _cvtsh_ss(arr[3]); - /* FALLTHRU */ - case 3: - out[2] = _cvtsh_ss(arr[2]); - /* FALLTHRU */ - case 2: - out[1] = _cvtsh_ss(arr[1]); - /* FALLTHRU */ - case 1: - out[0] = _cvtsh_ss(arr[0]); - } -} - -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float norm, float *out) { - const uint16_t *last = arr + size; - const uint16_t *last_aligned = arr + ((size >> 5) << 5); - __m512 zmm_norm = _mm512_set1_ps(norm); - - if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0x3f) == 0) { - for (; arr != last_aligned; arr += 32, out += 32) { - __m512 zmm_0 = _mm512_div_ps( - _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 0))), zmm_norm); - __m512 zmm_1 = _mm512_div_ps( - _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 16))), zmm_norm); - _mm512_store_ps(out + 0, zmm_0); - _mm512_store_ps(out + 16, zmm_1); - } - - if (last >= last_aligned + 16) { - _mm512_store_ps( - out, _mm512_div_ps(_mm512_cvtph_ps(_mm256_load_si256((__m256i *)arr)), - zmm_norm)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm256_store_ps( - out, _mm256_div_ps(_mm256_cvtph_ps(_mm_load_si128((__m128i *)arr)), - _mm256_set1_ps(norm))); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 32, out += 32) { - __m512 zmm_0 = _mm512_div_ps( - _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 0))), zmm_norm); - __m512 zmm_1 = _mm512_div_ps( - _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 16))), zmm_norm); - _mm512_storeu_ps(out + 0, zmm_0); - _mm512_storeu_ps(out + 16, zmm_1); - } - - if (last >= last_aligned + 16) { - _mm512_storeu_ps( - out, - _mm512_div_ps(_mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)arr)), - zmm_norm)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm256_storeu_ps( - out, _mm256_div_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr)), - _mm256_set1_ps(norm))); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtsh_ss(arr[6]) / norm; - /* FALLTHRU */ - case 6: - out[5] = _cvtsh_ss(arr[5]) / norm; - /* FALLTHRU */ - case 5: - out[4] = _cvtsh_ss(arr[4]) / norm; - /* FALLTHRU */ - case 4: - out[3] = _cvtsh_ss(arr[3]) / norm; - /* FALLTHRU */ - case 3: - out[2] = _cvtsh_ss(arr[2]) / norm; - /* FALLTHRU */ - case 2: - out[1] = _cvtsh_ss(arr[1]) / norm; - /* FALLTHRU */ - case 1: - out[0] = _cvtsh_ss(arr[0]) / norm; - } -} - -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - uint16_t *out) { - const float *last = arr + size; - const float *last_aligned = arr + ((size >> 5) << 5); - - if (((uintptr_t)arr & 0x3f) == 0 && ((uintptr_t)out & 0x1f) == 0) { - for (; arr != last_aligned; arr += 32, out += 32) { - _mm256_store_si256( - (__m256i *)(out + 0), - _mm512_cvtps_ph(_mm512_load_ps(arr + 0), _MM_FROUND_NO_EXC)); - _mm256_store_si256( - (__m256i *)(out + 16), - _mm512_cvtps_ph(_mm512_load_ps(arr + 16), _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 16) { - _mm256_store_si256( - (__m256i *)(out + 0), - _mm512_cvtps_ph(_mm512_load_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm_store_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 32, out += 32) { - _mm256_storeu_si256( - (__m256i *)(out + 0), - _mm512_cvtps_ph(_mm512_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); - _mm256_storeu_si256( - (__m256i *)(out + 16), - _mm512_cvtps_ph(_mm512_loadu_ps(arr + 16), _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 16) { - _mm256_storeu_si256( - (__m256i *)(out + 0), - _mm512_cvtps_ph(_mm512_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm_storeu_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtss_sh(arr[6], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 6: - out[5] = _cvtss_sh(arr[5], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 5: - out[4] = _cvtss_sh(arr[4], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 4: - out[3] = _cvtss_sh(arr[3], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 3: - out[2] = _cvtss_sh(arr[2], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 2: - out[1] = _cvtss_sh(arr[1], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 1: - out[0] = _cvtss_sh(arr[0], _MM_FROUND_NO_EXC); - } -} - -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - float norm, uint16_t *out) { - const float *last = arr + size; - const float *last_aligned = arr + ((size >> 5) << 5); - __m512 zmm_norm = _mm512_set1_ps(norm); - - if (((uintptr_t)arr & 0x3f) == 0 && ((uintptr_t)out & 0x1f) == 0) { - for (; arr != last_aligned; arr += 32, out += 32) { - __m512 zmm_0 = _mm512_div_ps(_mm512_load_ps(arr + 0), zmm_norm); - __m512 zmm_1 = _mm512_div_ps(_mm512_load_ps(arr + 16), zmm_norm); - _mm256_store_si256((__m256i *)(out + 0), - _mm512_cvtps_ph(zmm_0, _MM_FROUND_NO_EXC)); - _mm256_store_si256((__m256i *)(out + 16), - _mm512_cvtps_ph(zmm_1, _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 16) { - _mm256_store_si256( - (__m256i *)out, - _mm512_cvtps_ph(_mm512_div_ps(_mm512_load_ps(arr), zmm_norm), - _MM_FROUND_NO_EXC)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm_store_si128((__m128i *)out, - _mm256_cvtps_ph(_mm256_div_ps(_mm256_load_ps(arr), - _mm256_set1_ps(norm)), - _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 32, out += 32) { - __m512 zmm_0 = _mm512_div_ps(_mm512_loadu_ps(arr + 0), zmm_norm); - __m512 zmm_1 = _mm512_div_ps(_mm512_loadu_ps(arr + 16), zmm_norm); - _mm256_storeu_si256((__m256i *)(out + 0), - _mm512_cvtps_ph(zmm_0, _MM_FROUND_NO_EXC)); - _mm256_storeu_si256((__m256i *)(out + 16), - _mm512_cvtps_ph(zmm_1, _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 16) { - _mm256_storeu_si256( - (__m256i *)out, - _mm512_cvtps_ph(_mm512_div_ps(_mm512_loadu_ps(arr), zmm_norm), - _MM_FROUND_NO_EXC)); - arr += 16; - out += 16; - } - if (last >= arr + 8) { - _mm_storeu_si128((__m128i *)out, - _mm256_cvtps_ph(_mm256_div_ps(_mm256_loadu_ps(arr), - _mm256_set1_ps(norm)), - _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtss_sh(arr[6] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 6: - out[5] = _cvtss_sh(arr[5] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 5: - out[4] = _cvtss_sh(arr[4] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 4: - out[3] = _cvtss_sh(arr[3] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 3: - out[2] = _cvtss_sh(arr[2] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 2: - out[1] = _cvtss_sh(arr[1] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 1: - out[0] = _cvtss_sh(arr[0] / norm, _MM_FROUND_NO_EXC); - } -} -#elif defined(__F16C__) && defined(__AVX__) -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float *out) { - const uint16_t *last = arr + size; - const uint16_t *last_aligned = arr + ((size >> 4) << 4); - - if (((uintptr_t)arr & 0xf) == 0 && ((uintptr_t)out & 0x1f) == 0) { - for (; arr != last_aligned; arr += 16, out += 16) { - _mm256_store_ps(out + 0, - _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0)))); - _mm256_store_ps(out + 8, - _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 8)))); - } - - if (last >= last_aligned + 8) { - _mm256_store_ps(out + 0, - _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0)))); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 16, out += 16) { - _mm256_storeu_ps(out + 0, - _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0)))); - _mm256_storeu_ps(out + 8, - _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 8)))); - } - - if (last >= last_aligned + 8) { - _mm256_storeu_ps(out + 0, - _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0)))); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtsh_ss(arr[6]); - /* FALLTHRU */ - case 6: - out[5] = _cvtsh_ss(arr[5]); - /* FALLTHRU */ - case 5: - out[4] = _cvtsh_ss(arr[4]); - /* FALLTHRU */ - case 4: - out[3] = _cvtsh_ss(arr[3]); - /* FALLTHRU */ - case 3: - out[2] = _cvtsh_ss(arr[2]); - /* FALLTHRU */ - case 2: - out[1] = _cvtsh_ss(arr[1]); - /* FALLTHRU */ - case 1: - out[0] = _cvtsh_ss(arr[0]); - } -} - -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float norm, float *out) { - const uint16_t *last = arr + size; - const uint16_t *last_aligned = arr + ((size >> 4) << 4); - __m256 ymm_norm = _mm256_set1_ps(norm); - - if (((uintptr_t)arr & 0xf) == 0 && ((uintptr_t)out & 0x1f) == 0) { - for (; arr != last_aligned; arr += 16, out += 16) { - __m256 ymm_0 = _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0))); - __m256 ymm_1 = _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 8))); - ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); - ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); - _mm256_store_ps(out + 0, ymm_0); - _mm256_store_ps(out + 8, ymm_1); - } - - if (last >= last_aligned + 8) { - _mm256_store_ps( - out, _mm256_div_ps(_mm256_cvtph_ps(_mm_load_si128((__m128i *)arr)), - ymm_norm)); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 16, out += 16) { - __m256 ymm_0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0))); - __m256 ymm_1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 8))); - ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); - ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); - _mm256_storeu_ps(out + 0, ymm_0); - _mm256_storeu_ps(out + 8, ymm_1); - } - - if (last >= last_aligned + 8) { - _mm256_storeu_ps( - out, _mm256_div_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr)), - ymm_norm)); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtsh_ss(arr[6]) / norm; - /* FALLTHRU */ - case 6: - out[5] = _cvtsh_ss(arr[5]) / norm; - /* FALLTHRU */ - case 5: - out[4] = _cvtsh_ss(arr[4]) / norm; - /* FALLTHRU */ - case 4: - out[3] = _cvtsh_ss(arr[3]) / norm; - /* FALLTHRU */ - case 3: - out[2] = _cvtsh_ss(arr[2]) / norm; - /* FALLTHRU */ - case 2: - out[1] = _cvtsh_ss(arr[1]) / norm; - /* FALLTHRU */ - case 1: - out[0] = _cvtsh_ss(arr[0]) / norm; - } -} - -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - uint16_t *out) { - const float *last = arr + size; - const float *last_aligned = arr + ((size >> 4) << 4); - - if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0xf) == 0) { - for (; arr != last_aligned; arr += 16, out += 16) { - _mm_store_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); - _mm_store_si128( - (__m128i *)(out + 8), - _mm256_cvtps_ph(_mm256_load_ps(arr + 8), _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 8) { - _mm_store_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 16, out += 16) { - _mm_storeu_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); - _mm_storeu_si128( - (__m128i *)(out + 8), - _mm256_cvtps_ph(_mm256_loadu_ps(arr + 8), _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 8) { - _mm_storeu_si128( - (__m128i *)(out + 0), - _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtss_sh(arr[6], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 6: - out[5] = _cvtss_sh(arr[5], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 5: - out[4] = _cvtss_sh(arr[4], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 4: - out[3] = _cvtss_sh(arr[3], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 3: - out[2] = _cvtss_sh(arr[2], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 2: - out[1] = _cvtss_sh(arr[1], _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 1: - out[0] = _cvtss_sh(arr[0], _MM_FROUND_NO_EXC); - } -} - -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - float norm, uint16_t *out) { - const float *last = arr + size; - const float *last_aligned = arr + ((size >> 4) << 4); - __m256 ymm_norm = _mm256_set1_ps(norm); - - if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0xf) == 0) { - for (; arr != last_aligned; arr += 16, out += 16) { - __m256 ymm_0 = _mm256_load_ps(arr + 0); - __m256 ymm_1 = _mm256_load_ps(arr + 8); - ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); - ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); - _mm_store_si128((__m128i *)(out + 0), - _mm256_cvtps_ph(ymm_0, _MM_FROUND_NO_EXC)); - _mm_store_si128((__m128i *)(out + 8), - _mm256_cvtps_ph(ymm_1, _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 8) { - _mm_store_si128( - (__m128i *)out, - _mm256_cvtps_ph(_mm256_div_ps(_mm256_load_ps(arr), ymm_norm), - _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } else { - for (; arr != last_aligned; arr += 16, out += 16) { - __m256 ymm_0 = _mm256_loadu_ps(arr + 0); - __m256 ymm_1 = _mm256_loadu_ps(arr + 8); - ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); - ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); - _mm_storeu_si128((__m128i *)(out + 0), - _mm256_cvtps_ph(ymm_0, _MM_FROUND_NO_EXC)); - _mm_storeu_si128((__m128i *)(out + 8), - _mm256_cvtps_ph(ymm_1, _MM_FROUND_NO_EXC)); - } - - if (last >= last_aligned + 8) { - _mm_storeu_si128( - (__m128i *)out, - _mm256_cvtps_ph(_mm256_div_ps(_mm256_loadu_ps(arr), ymm_norm), - _MM_FROUND_NO_EXC)); - arr += 8; - out += 8; - } - } - switch (last - arr) { - case 7: - out[6] = _cvtss_sh(arr[6] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 6: - out[5] = _cvtss_sh(arr[5] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 5: - out[4] = _cvtss_sh(arr[4] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 4: - out[3] = _cvtss_sh(arr[3] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 3: - out[2] = _cvtss_sh(arr[2] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 2: - out[1] = _cvtss_sh(arr[1] / norm, _MM_FROUND_NO_EXC); - /* FALLTHRU */ - case 1: - out[0] = _cvtss_sh(arr[0] / norm, _MM_FROUND_NO_EXC); - } -} -#elif defined(__aarch64__) +#if defined(__aarch64__) static inline float float32(uint16_t val) { __fp16 *p = reinterpret_cast<__fp16 *>(&val); return *p; @@ -1095,35 +530,683 @@ static inline uint16_t float16(float val) { ((hbits & 0x7C00) != 0x7C00); return static_cast(hbits); } +#if defined(__F16C__) && defined(__AVX512F__) +static inline void convert_fp16_to_fp32_avx512f(const uint16_t *arr, + size_t size, float *out) { + const uint16_t *last = arr + size; + const uint16_t *last_aligned = arr + ((size >> 5) << 5); -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float *out) { + if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0x3f) == 0) { + for (; arr != last_aligned; arr += 32, out += 32) { + _mm512_store_ps(out + 0, + _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 0)))); + _mm512_store_ps( + out + 16, _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 16)))); + } + + if (last >= last_aligned + 16) { + _mm512_store_ps(out, _mm512_cvtph_ps(_mm256_load_si256((__m256i *)arr))); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm256_store_ps(out, _mm256_cvtph_ps(_mm_load_si128((__m128i *)arr))); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 32, out += 32) { + _mm512_storeu_ps( + out + 0, _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 0)))); + _mm512_storeu_ps( + out + 16, _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 16)))); + } + + if (last >= last_aligned + 16) { + _mm512_storeu_ps(out, + _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)arr))); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm256_storeu_ps(out, _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr))); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = float32(arr[6]); + /* FALLTHRU */ + case 6: + out[5] = float32(arr[5]); + /* FALLTHRU */ + case 5: + out[4] = float32(arr[4]); + /* FALLTHRU */ + case 4: + out[3] = float32(arr[3]); + /* FALLTHRU */ + case 3: + out[2] = float32(arr[2]); + /* FALLTHRU */ + case 2: + out[1] = float32(arr[1]); + /* FALLTHRU */ + case 1: + out[0] = float32(arr[0]); + } +} + +static inline void convert_fp16_to_fp32_avx512f(const uint16_t *arr, + size_t size, float norm, + float *out) { + const uint16_t *last = arr + size; + const uint16_t *last_aligned = arr + ((size >> 5) << 5); + __m512 zmm_norm = _mm512_set1_ps(norm); + + if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0x3f) == 0) { + for (; arr != last_aligned; arr += 32, out += 32) { + __m512 zmm_0 = _mm512_div_ps( + _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 0))), zmm_norm); + __m512 zmm_1 = _mm512_div_ps( + _mm512_cvtph_ps(_mm256_load_si256((__m256i *)(arr + 16))), zmm_norm); + _mm512_store_ps(out + 0, zmm_0); + _mm512_store_ps(out + 16, zmm_1); + } + + if (last >= last_aligned + 16) { + _mm512_store_ps( + out, _mm512_div_ps(_mm512_cvtph_ps(_mm256_load_si256((__m256i *)arr)), + zmm_norm)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm256_store_ps( + out, _mm256_div_ps(_mm256_cvtph_ps(_mm_load_si128((__m128i *)arr)), + _mm256_set1_ps(norm))); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 32, out += 32) { + __m512 zmm_0 = _mm512_div_ps( + _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 0))), zmm_norm); + __m512 zmm_1 = _mm512_div_ps( + _mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)(arr + 16))), zmm_norm); + _mm512_storeu_ps(out + 0, zmm_0); + _mm512_storeu_ps(out + 16, zmm_1); + } + + if (last >= last_aligned + 16) { + _mm512_storeu_ps( + out, + _mm512_div_ps(_mm512_cvtph_ps(_mm256_loadu_si256((__m256i *)arr)), + zmm_norm)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm256_storeu_ps( + out, _mm256_div_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr)), + _mm256_set1_ps(norm))); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = float32(arr[6]) / norm; + /* FALLTHRU */ + case 6: + out[5] = float32(arr[5]) / norm; + /* FALLTHRU */ + case 5: + out[4] = float32(arr[4]) / norm; + /* FALLTHRU */ + case 4: + out[3] = float32(arr[3]) / norm; + /* FALLTHRU */ + case 3: + out[2] = float32(arr[2]) / norm; + /* FALLTHRU */ + case 2: + out[1] = float32(arr[1]) / norm; + /* FALLTHRU */ + case 1: + out[0] = float32(arr[0]) / norm; + } +} + +static inline void convert_fp32_to_fp16_avx512f(const float *arr, size_t size, + uint16_t *out) { + const float *last = arr + size; + const float *last_aligned = arr + ((size >> 5) << 5); + + if (((uintptr_t)arr & 0x3f) == 0 && ((uintptr_t)out & 0x1f) == 0) { + for (; arr != last_aligned; arr += 32, out += 32) { + _mm256_store_si256( + (__m256i *)(out + 0), + _mm512_cvtps_ph(_mm512_load_ps(arr + 0), _MM_FROUND_NO_EXC)); + _mm256_store_si256( + (__m256i *)(out + 16), + _mm512_cvtps_ph(_mm512_load_ps(arr + 16), _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 16) { + _mm256_store_si256( + (__m256i *)(out + 0), + _mm512_cvtps_ph(_mm512_load_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm_store_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 32, out += 32) { + _mm256_storeu_si256( + (__m256i *)(out + 0), + _mm512_cvtps_ph(_mm512_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); + _mm256_storeu_si256( + (__m256i *)(out + 16), + _mm512_cvtps_ph(_mm512_loadu_ps(arr + 16), _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 16) { + _mm256_storeu_si256( + (__m256i *)(out + 0), + _mm512_cvtps_ph(_mm512_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm_storeu_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = float16(arr[6]); + /* FALLTHRU */ + case 6: + out[5] = float16(arr[5]); + /* FALLTHRU */ + case 5: + out[4] = float16(arr[4]); + /* FALLTHRU */ + case 4: + out[3] = float16(arr[3]); + /* FALLTHRU */ + case 3: + out[2] = float16(arr[2]); + /* FALLTHRU */ + case 2: + out[1] = float16(arr[1]); + /* FALLTHRU */ + case 1: + out[0] = float16(arr[0]); + } +} + +static inline void convert_fp32_to_fp16_avx512f(const float *arr, size_t size, + float norm, uint16_t *out) { + const float *last = arr + size; + const float *last_aligned = arr + ((size >> 5) << 5); + __m512 zmm_norm = _mm512_set1_ps(norm); + + if (((uintptr_t)arr & 0x3f) == 0 && ((uintptr_t)out & 0x1f) == 0) { + for (; arr != last_aligned; arr += 32, out += 32) { + __m512 zmm_0 = _mm512_div_ps(_mm512_load_ps(arr + 0), zmm_norm); + __m512 zmm_1 = _mm512_div_ps(_mm512_load_ps(arr + 16), zmm_norm); + _mm256_store_si256((__m256i *)(out + 0), + _mm512_cvtps_ph(zmm_0, _MM_FROUND_NO_EXC)); + _mm256_store_si256((__m256i *)(out + 16), + _mm512_cvtps_ph(zmm_1, _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 16) { + _mm256_store_si256( + (__m256i *)out, + _mm512_cvtps_ph(_mm512_div_ps(_mm512_load_ps(arr), zmm_norm), + _MM_FROUND_NO_EXC)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm_store_si128((__m128i *)out, + _mm256_cvtps_ph(_mm256_div_ps(_mm256_load_ps(arr), + _mm256_set1_ps(norm)), + _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 32, out += 32) { + __m512 zmm_0 = _mm512_div_ps(_mm512_loadu_ps(arr + 0), zmm_norm); + __m512 zmm_1 = _mm512_div_ps(_mm512_loadu_ps(arr + 16), zmm_norm); + _mm256_storeu_si256((__m256i *)(out + 0), + _mm512_cvtps_ph(zmm_0, _MM_FROUND_NO_EXC)); + _mm256_storeu_si256((__m256i *)(out + 16), + _mm512_cvtps_ph(zmm_1, _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 16) { + _mm256_storeu_si256( + (__m256i *)out, + _mm512_cvtps_ph(_mm512_div_ps(_mm512_loadu_ps(arr), zmm_norm), + _MM_FROUND_NO_EXC)); + arr += 16; + out += 16; + } + if (last >= arr + 8) { + _mm_storeu_si128((__m128i *)out, + _mm256_cvtps_ph(_mm256_div_ps(_mm256_loadu_ps(arr), + _mm256_set1_ps(norm)), + _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = float16(arr[6] / norm); + /* FALLTHRU */ + case 6: + out[5] = float16(arr[5] / norm); + /* FALLTHRU */ + case 5: + out[4] = float16(arr[4] / norm); + /* FALLTHRU */ + case 4: + out[3] = float16(arr[3] / norm); + /* FALLTHRU */ + case 3: + out[2] = float16(arr[2] / norm); + /* FALLTHRU */ + case 2: + out[1] = float16(arr[1] / norm); + /* FALLTHRU */ + case 1: + out[0] = float16(arr[0] / norm); + } +} +#endif //__F16C__ && __AVX512F__ + +#if defined(__F16C__) && defined(__AVX__) +static inline void convert_fp16_to_fp32_avx(const uint16_t *arr, size_t size, + float *out) { + const uint16_t *last = arr + size; + const uint16_t *last_aligned = arr + ((size >> 4) << 4); + + if (((uintptr_t)arr & 0xf) == 0 && ((uintptr_t)out & 0x1f) == 0) { + for (; arr != last_aligned; arr += 16, out += 16) { + _mm256_store_ps(out + 0, + _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0)))); + _mm256_store_ps(out + 8, + _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 8)))); + } + + if (last >= last_aligned + 8) { + _mm256_store_ps(out + 0, + _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0)))); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 16, out += 16) { + _mm256_storeu_ps(out + 0, + _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0)))); + _mm256_storeu_ps(out + 8, + _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 8)))); + } + + if (last >= last_aligned + 8) { + _mm256_storeu_ps(out + 0, + _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0)))); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = _cvtsh_ss(arr[6]); + /* FALLTHRU */ + case 6: + out[5] = _cvtsh_ss(arr[5]); + /* FALLTHRU */ + case 5: + out[4] = _cvtsh_ss(arr[4]); + /* FALLTHRU */ + case 4: + out[3] = _cvtsh_ss(arr[3]); + /* FALLTHRU */ + case 3: + out[2] = _cvtsh_ss(arr[2]); + /* FALLTHRU */ + case 2: + out[1] = _cvtsh_ss(arr[1]); + /* FALLTHRU */ + case 1: + out[0] = _cvtsh_ss(arr[0]); + } +} + +static inline void convert_fp16_to_fp32_avx(const uint16_t *arr, size_t size, + float norm, float *out) { + const uint16_t *last = arr + size; + const uint16_t *last_aligned = arr + ((size >> 4) << 4); + __m256 ymm_norm = _mm256_set1_ps(norm); + + if (((uintptr_t)arr & 0xf) == 0 && ((uintptr_t)out & 0x1f) == 0) { + for (; arr != last_aligned; arr += 16, out += 16) { + __m256 ymm_0 = _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 0))); + __m256 ymm_1 = _mm256_cvtph_ps(_mm_load_si128((__m128i *)(arr + 8))); + ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); + ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); + _mm256_store_ps(out + 0, ymm_0); + _mm256_store_ps(out + 8, ymm_1); + } + + if (last >= last_aligned + 8) { + _mm256_store_ps( + out, _mm256_div_ps(_mm256_cvtph_ps(_mm_load_si128((__m128i *)arr)), + ymm_norm)); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 16, out += 16) { + __m256 ymm_0 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 0))); + __m256 ymm_1 = _mm256_cvtph_ps(_mm_loadu_si128((__m128i *)(arr + 8))); + ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); + ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); + _mm256_storeu_ps(out + 0, ymm_0); + _mm256_storeu_ps(out + 8, ymm_1); + } + + if (last >= last_aligned + 8) { + _mm256_storeu_ps( + out, _mm256_div_ps(_mm256_cvtph_ps(_mm_loadu_si128((__m128i *)arr)), + ymm_norm)); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = _cvtsh_ss(arr[6]) / norm; + /* FALLTHRU */ + case 6: + out[5] = _cvtsh_ss(arr[5]) / norm; + /* FALLTHRU */ + case 5: + out[4] = _cvtsh_ss(arr[4]) / norm; + /* FALLTHRU */ + case 4: + out[3] = _cvtsh_ss(arr[3]) / norm; + /* FALLTHRU */ + case 3: + out[2] = _cvtsh_ss(arr[2]) / norm; + /* FALLTHRU */ + case 2: + out[1] = _cvtsh_ss(arr[1]) / norm; + /* FALLTHRU */ + case 1: + out[0] = _cvtsh_ss(arr[0]) / norm; + } +} + +static inline void convert_fp32_to_fp16_avx(const float *arr, size_t size, + uint16_t *out) { + const float *last = arr + size; + const float *last_aligned = arr + ((size >> 4) << 4); + + if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0xf) == 0) { + for (; arr != last_aligned; arr += 16, out += 16) { + _mm_store_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); + _mm_store_si128( + (__m128i *)(out + 8), + _mm256_cvtps_ph(_mm256_load_ps(arr + 8), _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 8) { + _mm_store_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_load_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 16, out += 16) { + _mm_storeu_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); + _mm_storeu_si128( + (__m128i *)(out + 8), + _mm256_cvtps_ph(_mm256_loadu_ps(arr + 8), _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 8) { + _mm_storeu_si128( + (__m128i *)(out + 0), + _mm256_cvtps_ph(_mm256_loadu_ps(arr + 0), _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = _cvtss_sh(arr[6], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 6: + out[5] = _cvtss_sh(arr[5], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 5: + out[4] = _cvtss_sh(arr[4], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 4: + out[3] = _cvtss_sh(arr[3], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 3: + out[2] = _cvtss_sh(arr[2], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 2: + out[1] = _cvtss_sh(arr[1], _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 1: + out[0] = _cvtss_sh(arr[0], _MM_FROUND_NO_EXC); + } +} + +static inline void convert_fp32_to_fp16_avx(const float *arr, size_t size, + float norm, uint16_t *out) { + const float *last = arr + size; + const float *last_aligned = arr + ((size >> 4) << 4); + __m256 ymm_norm = _mm256_set1_ps(norm); + + if (((uintptr_t)arr & 0x1f) == 0 && ((uintptr_t)out & 0xf) == 0) { + for (; arr != last_aligned; arr += 16, out += 16) { + __m256 ymm_0 = _mm256_load_ps(arr + 0); + __m256 ymm_1 = _mm256_load_ps(arr + 8); + ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); + ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); + _mm_store_si128((__m128i *)(out + 0), + _mm256_cvtps_ph(ymm_0, _MM_FROUND_NO_EXC)); + _mm_store_si128((__m128i *)(out + 8), + _mm256_cvtps_ph(ymm_1, _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 8) { + _mm_store_si128( + (__m128i *)out, + _mm256_cvtps_ph(_mm256_div_ps(_mm256_load_ps(arr), ymm_norm), + _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } else { + for (; arr != last_aligned; arr += 16, out += 16) { + __m256 ymm_0 = _mm256_loadu_ps(arr + 0); + __m256 ymm_1 = _mm256_loadu_ps(arr + 8); + ymm_0 = _mm256_div_ps(ymm_0, ymm_norm); + ymm_1 = _mm256_div_ps(ymm_1, ymm_norm); + _mm_storeu_si128((__m128i *)(out + 0), + _mm256_cvtps_ph(ymm_0, _MM_FROUND_NO_EXC)); + _mm_storeu_si128((__m128i *)(out + 8), + _mm256_cvtps_ph(ymm_1, _MM_FROUND_NO_EXC)); + } + + if (last >= last_aligned + 8) { + _mm_storeu_si128( + (__m128i *)out, + _mm256_cvtps_ph(_mm256_div_ps(_mm256_loadu_ps(arr), ymm_norm), + _MM_FROUND_NO_EXC)); + arr += 8; + out += 8; + } + } + switch (last - arr) { + case 7: + out[6] = _cvtss_sh(arr[6] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 6: + out[5] = _cvtss_sh(arr[5] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 5: + out[4] = _cvtss_sh(arr[4] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 4: + out[3] = _cvtss_sh(arr[3] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 3: + out[2] = _cvtss_sh(arr[2] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 2: + out[1] = _cvtss_sh(arr[1] / norm, _MM_FROUND_NO_EXC); + /* FALLTHRU */ + case 1: + out[0] = _cvtss_sh(arr[0] / norm, _MM_FROUND_NO_EXC); + } +} +#endif // __F16C__ && __AVX__ + +static inline void convert_fp16_to_fp32_fallback(const uint16_t *arr, + size_t size, float *out) { for (size_t i = 0; i != size; ++i) { out[i] = float32(arr[i]); } } -static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, - float norm, float *out) { +static inline void convert_fp16_to_fp32_fallback(const uint16_t *arr, + size_t size, float norm, + float *out) { for (size_t i = 0; i != size; ++i) { out[i] = float32(arr[i]) / norm; } } -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - uint16_t *out) { +static inline void convert_fp32_to_fp16_fallback(const float *arr, size_t size, + uint16_t *out) { for (size_t i = 0; i != size; ++i) { out[i] = float16(arr[i]); } } -static inline void convert_fp32_to_fp16(const float *arr, size_t size, - float norm, uint16_t *out) { +static inline void convert_fp32_to_fp16_fallback(const float *arr, size_t size, + float norm, uint16_t *out) { for (size_t i = 0; i != size; ++i) { out[i] = float16(arr[i] / norm); } } -#endif // __F16C__ && __AVX512F__ + +static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, + float *out) { +#if defined(__F16C__) && defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return convert_fp16_to_fp32_avx512f(arr, size, out); + } +#endif + +#if defined(__F16C__) && defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + return convert_fp16_to_fp32_avx(arr, size, out); + } +#endif + + return convert_fp16_to_fp32_fallback(arr, size, out); +} + +static inline void convert_fp16_to_fp32(const uint16_t *arr, size_t size, + float norm, float *out) { +#if defined(__F16C__) && defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return convert_fp16_to_fp32_avx512f(arr, size, norm, out); + } +#endif + +#if defined(__F16C__) && defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + return convert_fp16_to_fp32_avx(arr, size, norm, out); + } +#endif + + return convert_fp16_to_fp32_fallback(arr, size, norm, out); +} + +static inline void convert_fp32_to_fp16(const float *arr, size_t size, + uint16_t *out) { +#if defined(__F16C__) && defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return convert_fp32_to_fp16_avx512f(arr, size, out); + } +#endif + +#if defined(__F16C__) && defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + return convert_fp32_to_fp16_avx(arr, size, out); + } +#endif + + return convert_fp32_to_fp16_fallback(arr, size, out); +} + +static inline void convert_fp32_to_fp16(const float *arr, size_t size, + float norm, uint16_t *out) { +#if defined(__F16C__) && defined(__AVX512F__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) { + return convert_fp32_to_fp16_avx512f(arr, size, norm, out); + } +#endif + +#if defined(__F16C__) && defined(__AVX__) + if (zvec::ailego::internal::CpuFeatures::static_flags_.F16C && + zvec::ailego::internal::CpuFeatures::static_flags_.AVX) { + return convert_fp32_to_fp16_avx(arr, size, norm, out); + } +#endif + + return convert_fp32_to_fp16_fallback(arr, size, norm, out); +} + +#endif // namespace zvec { namespace ailego { @@ -1155,4 +1238,4 @@ void FloatHelper::ToFP16(const float *arr, size_t size, float norm, } } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/tests/ailego/internal/cpu_features_test.cc b/tests/ailego/internal/cpu_features_test.cc index 37ae43f..2872321 100644 --- a/tests/ailego/internal/cpu_features_test.cc +++ b/tests/ailego/internal/cpu_features_test.cc @@ -176,3 +176,111 @@ TEST(CpuFeatures, General) { EXPECT_TRUE(CpuFeatures::MMX()); #endif } + + +TEST(CpuFeatures, Static) { + std::cout << "* F16C: " << CpuFeatures::static_flags_.F16C + << std::endl; + std::cout << "* SSE: " << CpuFeatures::static_flags_.SSE + << std::endl; + std::cout << "* SSE2: " << CpuFeatures::static_flags_.SSE2 + << std::endl; + std::cout << "* SSE3: " << CpuFeatures::static_flags_.SSE3 + << std::endl; + std::cout << "* SSSE3: " << CpuFeatures::static_flags_.SSSE3 + << std::endl; + std::cout << "* SSE4_1: " << CpuFeatures::static_flags_.SSE4_1 + << std::endl; + std::cout << "* SSE4_2: " << CpuFeatures::static_flags_.SSE4_2 + << std::endl; + std::cout << "* AVX: " << CpuFeatures::static_flags_.AVX + << std::endl; + std::cout << "* AVX2: " << CpuFeatures::static_flags_.AVX2 + << std::endl; + std::cout << "* AVX512F: " << CpuFeatures::static_flags_.AVX512F + << std::endl; + std::cout << "* AVX512DQ: " << CpuFeatures::static_flags_.AVX512DQ + << std::endl; + std::cout << "* AVX512PF: " << CpuFeatures::static_flags_.AVX512PF + << std::endl; + std::cout << "* AVX512ER: " << CpuFeatures::static_flags_.AVX512ER + << std::endl; + std::cout << "* AVX512CD: " << CpuFeatures::static_flags_.AVX512CD + << std::endl; + std::cout << "* AVX512BW: " << CpuFeatures::static_flags_.AVX512BW + << std::endl; + std::cout << "* AVX512VL: " << CpuFeatures::static_flags_.AVX512VL + << std::endl; + std::cout << "* AVX512_IFMA: " << CpuFeatures::static_flags_.AVX512_IFMA + << std::endl; + std::cout << "* AVX512_VBMI: " << CpuFeatures::static_flags_.AVX512_VBMI + << std::endl; + std::cout << "* AVX512_VBMI2: " << CpuFeatures::static_flags_.AVX512_VBMI2 + << std::endl; + std::cout << "* AVX512_VNNI: " << CpuFeatures::static_flags_.AVX512_VNNI + << std::endl; + std::cout << "* AVX512_BITALG: " + << CpuFeatures::static_flags_.AVX512_BITALG << std::endl; + std::cout << "* AVX512_VPOPCNTDQ: " + << CpuFeatures::static_flags_.AVX512_VPOPCNTDQ << std::endl; + std::cout << "* AVX512_4VNNIW: " + << CpuFeatures::static_flags_.AVX512_4VNNIW << std::endl; + std::cout << "* AVX512_4FMAPS: " + << CpuFeatures::static_flags_.AVX512_4FMAPS << std::endl; + std::cout << "* AVX512_FP16: " << CpuFeatures::static_flags_.AVX512_FP16 + << std::endl; + std::cout << "* CX8: " << CpuFeatures::static_flags_.CX8 + << std::endl; + std::cout << "* CX16: " << CpuFeatures::static_flags_.CX16 + << std::endl; + std::cout << "* PCLMULQDQ: " << CpuFeatures::static_flags_.PCLMULQDQ + << std::endl; + std::cout << "* VPCLMULQDQ: " << CpuFeatures::static_flags_.VPCLMULQDQ + << std::endl; + std::cout << "* CMOV: " << CpuFeatures::static_flags_.CMOV + << std::endl; + std::cout << "* MOVBE: " << CpuFeatures::static_flags_.MOVBE + << std::endl; + std::cout << "* ERMS: " << CpuFeatures::static_flags_.ERMS + << std::endl; + std::cout << "* POPCNT: " << CpuFeatures::static_flags_.POPCNT + << std::endl; + std::cout << "* XSAVE: " << CpuFeatures::static_flags_.XSAVE + << std::endl; + std::cout << "* FMA: " << CpuFeatures::static_flags_.FMA + << std::endl; + std::cout << "* ADX: " << CpuFeatures::static_flags_.ADX + << std::endl; + std::cout << "* GFNI: " << CpuFeatures::static_flags_.GFNI + << std::endl; + std::cout << "* AES: " << CpuFeatures::static_flags_.AES + << std::endl; + std::cout << "* VAES: " << CpuFeatures::static_flags_.VAES + << std::endl; + std::cout << "* RDSEED: " << CpuFeatures::static_flags_.RDSEED + << std::endl; + std::cout << "* RDRAND: " << CpuFeatures::static_flags_.RDRAND + << std::endl; + std::cout << "* SHA: " << CpuFeatures::static_flags_.SHA + << std::endl; + std::cout << "* BMI1: " << CpuFeatures::static_flags_.BMI1 + << std::endl; + std::cout << "* BMI2: " << CpuFeatures::static_flags_.BMI2 + << std::endl; + std::cout << "* CLFLUSH: " << CpuFeatures::static_flags_.CLFLUSH + << std::endl; + std::cout << "* CLFLUSHOPT: " << CpuFeatures::static_flags_.CLFLUSHOPT + << std::endl; + std::cout << "* CLWB: " << CpuFeatures::static_flags_.CLWB + << std::endl; + std::cout << "* RDPID: " << CpuFeatures::static_flags_.RDPID + << std::endl; + std::cout << "* FPU: " << CpuFeatures::static_flags_.FPU + << std::endl; + std::cout << "* HT: " << CpuFeatures::static_flags_.HT + << std::endl; + std::cout << "* VMX: " << CpuFeatures::static_flags_.VMX + << std::endl; + std::cout << "* HYPERVISOR: " << CpuFeatures::static_flags_.HYPERVISOR + << std::endl; +} \ No newline at end of file