xiaoxue_admin
/
zvec
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(core): support cpu flag detect & dispatch (#3)

This commit is contained in:
feihongxu0824 2025-12-30 18:42:47 +08:00 committed by GitHub
parent 220a9aba11
commit 1d9791b57a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 1233 additions and 764 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/ailego/internal/cpu_features.cc
View File

@ -416,6 +416,7 @@ const char *CpuFeatures::Intrinsics(void) {
;
}
CpuFeatures::StaticFlags CpuFeatures::static_flags_;
} // namespace internal
} // namespace ailego
} // namespace zvec

163
src/ailego/internal/cpu_features.h
View File

@ -201,6 +201,169 @@ class CpuFeatures {
//! Static Members
static CpuFlags flags_;
public:
struct StaticFlags {
//! 16-bit FP conversions
bool F16C = CpuFeatures::F16C();
//! Multimedia Extensions
bool MMX = CpuFeatures::MMX();
//! Streaming SIMD Extensions
bool SSE = CpuFeatures::SSE();
//! Streaming SIMD Extensions 2
bool SSE2 = CpuFeatures::SSE2();
//! Streaming SIMD Extensions 3
bool SSE3 = CpuFeatures::SSE3();
//! Supplemental Streaming SIMD Extensions 3
bool SSSE3 = CpuFeatures::SSSE3();
//! Streaming SIMD Extensions 4.1
bool SSE4_1 = CpuFeatures::SSE4_1();
//! Streaming SIMD Extensions 4.2
bool SSE4_2 = CpuFeatures::SSE4_2();
//! Advanced Vector Extensions
bool AVX = CpuFeatures::AVX();
//! Advanced Vector Extensions 2
bool AVX2 = CpuFeatures::AVX2();
//! AVX-512 Foundation
bool AVX512F = CpuFeatures::AVX512F();
//! AVX-512 DQ (Double/Quad granular) Instructions
bool AVX512DQ = CpuFeatures::AVX512DQ();
//! AVX-512 Prefetch
bool AVX512PF = CpuFeatures::AVX512PF();
//! AVX-512 Exponential and Reciprocal
bool AVX512ER = CpuFeatures::AVX512ER();
//! AVX-512 Conflict Detection
bool AVX512CD = CpuFeatures::AVX512CD();
//! AVX-512 BW (Byte/Word granular) Instructions
bool AVX512BW = CpuFeatures::AVX512BW();
//! AVX-512 VL (128/256 Vector Length) Extensions
bool AVX512VL = CpuFeatures::AVX512VL();
//! AVX-512 Integer Fused Multiply-Add instructions
bool AVX512_IFMA = CpuFeatures::AVX512_IFMA();
//! AVX512 Vector Bit Manipulation instructions
bool AVX512_VBMI = CpuFeatures::AVX512_VBMI();
//! Additional AVX512 Vector Bit Manipulation Instructions
bool AVX512_VBMI2 = CpuFeatures::AVX512_VBMI2();
//! Vector Neural Network Instructions
bool AVX512_VNNI = CpuFeatures::AVX512_VNNI();
//! Support for VPOPCNT[B,W] and VPSHUF-BITQMB instructions
bool AVX512_BITALG = CpuFeatures::AVX512_BITALG();
//! POPCNT for vectors of DW/QW
bool AVX512_VPOPCNTDQ = CpuFeatures::AVX512_VPOPCNTDQ();
//! AVX-512 Neural Network Instructions
bool AVX512_4VNNIW = CpuFeatures::AVX512_4VNNIW();
//! AVX-512 Multiply Accumulation Single precision
bool AVX512_4FMAPS = CpuFeatures::AVX512_4FMAPS();
//! AVX-512 FP16 instructions
bool AVX512_FP16 = CpuFeatures::AVX512_FP16();
//! CMPXCHG8 instruction
bool CX8 = CpuFeatures::CX8();
//! CMPXCHG16B instruction
bool CX16 = CpuFeatures::CX16();
//! PCLMULQDQ instruction
bool PCLMULQDQ = CpuFeatures::PCLMULQDQ();
//! Carry-Less Multiplication Double Quadword
bool VPCLMULQDQ = CpuFeatures::VPCLMULQDQ();
//! CMOV instructions (plus FCMOVcc, FCOMI with FPU)
bool CMOV = CpuFeatures::CMOV();
//! MOVBE instruction
bool MOVBE = CpuFeatures::MOVBE();
//! Enhanced REP MOVSB/STOSB instructions
bool ERMS = CpuFeatures::ERMS();
//! POPCNT instruction
bool POPCNT = CpuFeatures::POPCNT();
//! XSAVE/XRSTOR/XSETBV/XGETBV instructions
bool XSAVE = CpuFeatures::XSAVE();
//! Fused multiply-add
bool FMA = CpuFeatures::FMA();
//! ADCX and ADOX instructions
bool ADX = CpuFeatures::ADX();
//! Galois Field New Instructions
bool GFNI = CpuFeatures::GFNI();
//! AES instructions
bool AES = CpuFeatures::AES();
//! Vector AES
bool VAES = CpuFeatures::VAES();
//! RDSEED instruction
bool RDSEED = CpuFeatures::RDSEED();
//! RDRAND instruction
bool RDRAND = CpuFeatures::RDRAND();
//! SHA1/SHA256 Instruction Extensions
bool SHA = CpuFeatures::SHA();
//! 1st group bit manipulation extensions
bool BMI1 = CpuFeatures::BMI1();
//! 2nd group bit manipulation extensions
bool BMI2 = CpuFeatures::BMI2();
//! CLFLUSH instruction
bool CLFLUSH = CpuFeatures::CLFLUSH();
//! CLFLUSHOPT instruction
bool CLFLUSHOPT = CpuFeatures::CLFLUSHOPT();
//! CLWB instruction
bool CLWB = CpuFeatures::CLWB();
//! RDPID instruction
bool RDPID = CpuFeatures::RDPID();
//! Onboard FPU
bool FPU = CpuFeatures::FPU();
//! Hyper-Threading
bool HT = CpuFeatures::HT();
//! Hardware virtualization
bool VMX = CpuFeatures::VMX();
// ！Running on a hypervisor
bool HYPERVISOR = CpuFeatures::HYPERVISOR();
};
static StaticFlags static_flags_;
};
} // namespace internal

10
src/ailego/math/distance_matrix_fp16.i
View File

@ -14,7 +14,7 @@
#include <ailego/internal/platform.h>
#include "matrix_define.i"
#include <iostream>
#if !defined(__AVX__)
#define _mm_broadcast_si32(a) _mm_castps_si128(_mm_load1_ps((const float *)(a)))
#else
@ -95,13 +95,13 @@
} \
case 1: { \
__m256 ymm_lhs = _mm256_cvtph_ps( \
_mm_set_epi16((short)(_MASK), (short)(_MASK), (short)(_MASK), \
_mm_set_epi16(*((const short *)(lhs)), (short)(_MASK), \
(short)(_MASK), (short)(_MASK), (short)(_MASK), \
(short)(_MASK), *((const short *)(lhs)))); \
(short)(_MASK), (short)(_MASK), (short)(_MASK))); \
__m256 ymm_rhs = _mm256_cvtph_ps( \
_mm_set_epi16((short)(_MASK), (short)(_MASK), (short)(_MASK), \
_mm_set_epi16(*((const short *)(rhs)), (short)(_MASK), \
(short)(_MASK), (short)(_MASK), (short)(_MASK), \
(short)(_MASK), *((const short *)(rhs)))); \
(short)(_MASK), (short)(_MASK), (short)(_MASK))); \
_PROC(ymm_lhs, ymm_rhs, _RES##_0_0) \
break; \
} \

34
src/ailego/math/euclidean_distance_matrix.h
View File

@ -178,14 +178,15 @@ struct SquaredEuclideanDistanceMatrix<
protected:
//! Calculate the squared difference
static inline float SquaredDifference(uint32_t lhs, uint32_t rhs) {
return static_cast<float>(MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 24), (int8_t)(rhs >> 24)));
volatile int32_t sum = MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 24), (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};
@ -228,14 +229,15 @@ struct SquaredEuclideanDistanceMatrix<int8_t, M, 1,
protected:
//! Calculate the squared difference
static inline float SquaredDifference(uint32_t lhs, uint32_t rhs) {
return static_cast<float>(MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 24), (int8_t)(rhs >> 24)));
volatile int32_t sum = MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 0), (int8_t)(rhs >> 0)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 8), (int8_t)(rhs >> 8)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 16), (int8_t)(rhs >> 16)) +
MathHelper::SquaredDifference<int8_t, int32_t>(
(int8_t)(lhs >> 24), (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};

216
src/ailego/math/euclidean_distance_matrix_fp16.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ailego/internal/cpu_features.h>
#include "distance_matrix_accum_fp16.i"
#include "euclidean_distance_matrix.h"
@ -131,6 +132,7 @@ static inline float SquaredEuclideanDistanceAVX512FP16(const Float16 *lhs,
}
#endif
//! Compute the distance between matrix and query (FP16, M=1, N=1)
void SquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
const ValueType *q,
@ -138,13 +140,21 @@ void SquaredEuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
float *out) {
#if defined(__ARM_NEON)
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, )
#elif defined(__AVX512FP16__)
*out = SquaredEuclideanDistanceAVX512FP16(m, q, dim);
#elif defined(__AVX512F__)
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
#else
#if defined(__AVX512FP16__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
*out = SquaredEuclideanDistanceAVX512FP16(m, q, dim);
return;
}
#endif
#if defined(__AVX512F__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
return;
}
#endif
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, )
#endif // __AVX512F__
#endif //__ARM_NEON
}
//! Compute the distance between matrix and query (FP16, M=1, N=1)
@ -153,13 +163,21 @@ void EuclideanDistanceMatrix<Float16, 1, 1>::Compute(const ValueType *m,
size_t dim, float *out) {
#if defined(__ARM_NEON)
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, std::sqrt)
#elif defined(__AVX512FP16__)
*out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim));
#elif defined(__AVX512F__)
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt)
#else
#if defined(__AVX512FP16__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
*out = std::sqrt(SquaredEuclideanDistanceAVX512FP16(m, q, dim));
return;
}
#endif
#if defined(__AVX512F__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, std::sqrt)
return;
}
#endif
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, std::sqrt)
#endif // __AVX512F__
#endif //__ARM_NEON
}
#if !defined(__ARM_NEON)
@ -241,10 +259,13 @@ void SquaredEuclideanDistanceMatrix<Float16, 16, 1>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X1_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_16X1_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X1_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X1_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=16, N=2)
@ -253,10 +274,13 @@ void SquaredEuclideanDistanceMatrix<Float16, 16, 2>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X2_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_16X2_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X2_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X2_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=16, N=4)
@ -265,10 +289,12 @@ void SquaredEuclideanDistanceMatrix<Float16, 16, 4>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X4_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_16X4_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X4_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X4_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=16, N=8)
@ -277,20 +303,24 @@ void SquaredEuclideanDistanceMatrix<Float16, 16, 8>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X8_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_16X8_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X8_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X8_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=16, N=16)
void SquaredEuclideanDistanceMatrix<Float16, 16, 16>::Compute(
const ValueType *m, const ValueType *q, size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X16_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_16X16_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X16_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X16_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=1)
@ -299,10 +329,12 @@ void SquaredEuclideanDistanceMatrix<Float16, 32, 1>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X1_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X1_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X1_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X1_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=2)
@ -311,10 +343,12 @@ void SquaredEuclideanDistanceMatrix<Float16, 32, 2>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X2_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X2_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X2_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X2_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=4)
@ -323,10 +357,12 @@ void SquaredEuclideanDistanceMatrix<Float16, 32, 4>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X4_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X4_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X4_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X4_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=8)
@ -335,30 +371,36 @@ void SquaredEuclideanDistanceMatrix<Float16, 32, 8>::Compute(const ValueType *m,
size_t dim,
float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X8_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X8_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X8_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X8_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=16)
void SquaredEuclideanDistanceMatrix<Float16, 32, 16>::Compute(
const ValueType *m, const ValueType *q, size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X16_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X16_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X16_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X16_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=32, N=32)
void SquaredEuclideanDistanceMatrix<Float16, 32, 32>::Compute(
const ValueType *m, const ValueType *q, size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X32_AVX512(m, q, dim, out, )
#else
ACCUM_FP16_32X32_AVX(m, q, dim, out, )
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X32_AVX512(m, q, dim, out, )
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X32_AVX(m, q, dim, out, )
}
//! Compute the distance between matrix and query (FP16, M=2, N=1)
@ -429,10 +471,12 @@ void EuclideanDistanceMatrix<Float16, 16, 1>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=16, N=2)
@ -440,10 +484,12 @@ void EuclideanDistanceMatrix<Float16, 16, 2>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=16, N=4)
@ -451,10 +497,12 @@ void EuclideanDistanceMatrix<Float16, 16, 4>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=16, N=8)
@ -462,10 +510,12 @@ void EuclideanDistanceMatrix<Float16, 16, 8>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=16, N=16)
@ -473,10 +523,12 @@ void EuclideanDistanceMatrix<Float16, 16, 16>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_16X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_16X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=1)
@ -484,10 +536,12 @@ void EuclideanDistanceMatrix<Float16, 32, 1>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X1_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X1_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=2)
@ -495,10 +549,12 @@ void EuclideanDistanceMatrix<Float16, 32, 2>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X2_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X2_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=4)
@ -506,10 +562,12 @@ void EuclideanDistanceMatrix<Float16, 32, 4>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X4_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X4_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=8)
@ -517,10 +575,12 @@ void EuclideanDistanceMatrix<Float16, 32, 8>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X8_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X8_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=16)
@ -528,10 +588,12 @@ void EuclideanDistanceMatrix<Float16, 32, 16>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X16_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X16_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
//! Compute the distance between matrix and query (FP16, M=32, N=32)
@ -539,10 +601,12 @@ void EuclideanDistanceMatrix<Float16, 32, 32>::Compute(const ValueType *m,
const ValueType *q,
size_t dim, float *out) {
#if defined(__AVX512F__)
ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps)
#else
ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_32X32_AVX512(m, q, dim, out, _mm512_sqrt_ps)
return;
}
#endif // __AVX512F__
ACCUM_FP16_32X32_AVX(m, q, dim, out, _mm256_sqrt_ps)
}
#endif // !__ARM_NEON
#endif // (__F16C__ && __AVX__) || (__ARM_NEON && __aarch64__)

34
src/ailego/math/euclidean_distance_matrix_fp32.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ailego/internal/cpu_features.h>
#include "distance_matrix_accum_fp32.i"
#include "euclidean_distance_matrix.h"
@ -291,15 +292,19 @@ void SquaredEuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
*out = SquaredEuclideanDistanceNEON(m, q, dim);
#else
#if defined(__AVX512F__)
if (dim > 15) {
*out = SquaredEuclideanDistanceAVX512(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
if (dim > 15) {
*out = SquaredEuclideanDistanceAVX512(m, q, dim);
return;
}
}
#endif // __AVX512F__
#if defined(__AVX__)
if (dim > 7) {
*out = SquaredEuclideanDistanceAVX(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
if (dim > 7) {
*out = SquaredEuclideanDistanceAVX(m, q, dim);
return;
}
}
#endif // __AVX__
*out = SquaredEuclideanDistanceSSE(m, q, dim);
@ -618,15 +623,20 @@ void EuclideanDistanceMatrix<float, 1, 1>::Compute(const ValueType *m,
*out = std::sqrt(SquaredEuclideanDistanceNEON(m, q, dim));
#else
#if defined(__AVX512F__)
if (dim > 15) {
*out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim));
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
if (dim > 15) {
*out = std::sqrt(SquaredEuclideanDistanceAVX512(m, q, dim));
return;
}
}
#endif // __AVX512F__
#if defined(__AVX__)
if (dim > 7) {
*out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim));
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
if (dim > 7) {
*out = std::sqrt(SquaredEuclideanDistanceAVX(m, q, dim));
return;
}
}
#endif // __AVX__
*out = std::sqrt(SquaredEuclideanDistanceSSE(m, q, dim));

40
src/ailego/math/inner_product_matrix.h
View File

@ -181,10 +181,12 @@ struct InnerProductMatrix<int8_t, M, N,
protected:
//! Calculate Fused-Multiply-Add
static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) {
return static_cast<float>((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};
@ -226,10 +228,12 @@ struct InnerProductMatrix<int8_t, M, 1, typename std::enable_if<M >= 2>::type> {
protected:
//! Calculate Fused-Multiply-Add
static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) {
return static_cast<float>((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};
@ -524,10 +528,12 @@ struct MinusInnerProductMatrix<
protected:
//! Calculate Fused-Multiply-Add
static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) {
return static_cast<float>((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};
@ -570,10 +576,12 @@ struct MinusInnerProductMatrix<int8_t, M, 1,
protected:
//! Calculate Fused-Multiply-Add
static inline float FusedMultiplyAdd(uint32_t lhs, uint32_t rhs) {
return static_cast<float>((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
volatile int32_t sum = ((int8_t)(lhs >> 0) * (int8_t)(rhs >> 0) +
(int8_t)(lhs >> 8) * (int8_t)(rhs >> 8) +
(int8_t)(lhs >> 16) * (int8_t)(rhs >> 16) +
(int8_t)(lhs >> 24) * (int8_t)(rhs >> 24));
return static_cast<float>(sum);
}
};

37
src/ailego/math/inner_product_matrix_fp16.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ailego/internal/cpu_features.h>
#include "distance_matrix_accum_fp16.i"
#include "inner_product_matrix.h"
@ -144,13 +145,21 @@ void InnerProductMatrix<Float16, 1, 1>::Compute(const ValueType *m,
float *out) {
#if defined(__ARM_NEON)
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, )
#elif defined(__AVX512FP16__)
*out = InnerProductAVX512FP16(m, q, dim);
#elif defined(__AVX512F__)
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
#else
#if defined(__AVX512FP16__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
*out = InnerProductAVX512FP16(m, q, dim);
return;
}
#endif //__AVX512FP16__
#if defined(__AVX512F__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, )
return;
}
#endif //__AVX512F__
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, )
#endif
#endif //__ARM_NEON
}
//! Compute the distance between matrix and query (FP16, M=1, N=1)
@ -159,13 +168,21 @@ void MinusInnerProductMatrix<Float16, 1, 1>::Compute(const ValueType *m,
size_t dim, float *out) {
#if defined(__ARM_NEON)
ACCUM_FP16_1X1_NEON(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL)
#elif defined(__AVX512FP16__)
*out = -InnerProductAVX512FP16(m, q, dim);
#elif defined(__AVX512F__)
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL)
#else
#if defined(__AVX512FP16__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_FP16) {
*out = -InnerProductAVX512FP16(m, q, dim);
return;
}
#endif //__AVX512FP16__
#if defined(__AVX512F__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
ACCUM_FP16_1X1_AVX512(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL)
return;
}
#endif //__AVX512F__
ACCUM_FP16_1X1_AVX(m, q, dim, out, 0ull, NEGATE_FP32_GENERAL)
#endif
#endif //__ARM_NEON
}
#if !defined(__ARM_NEON)

54
src/ailego/math/inner_product_matrix_fp32.cc
View File

@ -12,30 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ailego/internal/cpu_features.h>
#include "distance_matrix_accum_fp32.i"
#include "inner_product_matrix.h"
#if DEBUG_PRINT
#include <iostream>
static void inline print_data16(__m128i *data) {
uint16_t buffer[16];
memcpy(buffer, data, sizeof(buffer));
// std::cout << "result equals: " << std::endl;
for (int i = 0; i < 8; i++) {
uint64_t value = buffer[i];
if (i < 8) {
std::cout << value << ", ";
} else {
std::cout << value << std::endl;
}
}
}
#endif
namespace zvec {
namespace ailego {
@ -585,15 +565,19 @@ void InnerProductMatrix<float, 1, 1>::Compute(const ValueType *m,
*out = InnerProductNEON(m, q, dim);
#else
#if defined(__AVX512F__)
if (dim > 15) {
*out = InnerProductAVX512(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
if (dim > 15) {
*out = InnerProductAVX512(m, q, dim);
return;
}
}
#endif // __AVX512F__
#if defined(__AVX__)
if (dim > 7) {
*out = InnerProductAVX(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
if (dim > 7) {
*out = InnerProductAVX(m, q, dim);
return;
}
}
#endif // __AVX__
*out = InnerProductSSE(m, q, dim);
@ -890,15 +874,19 @@ void MinusInnerProductMatrix<float, 1, 1>::Compute(const ValueType *m,
*out = -InnerProductNEON(m, q, dim);
#else
#if defined(__AVX512F__)
if (dim > 15) {
*out = -InnerProductAVX512(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
if (dim > 15) {
*out = -InnerProductAVX512(m, q, dim);
return;
}
}
#endif // __AVX512F__
#if defined(__AVX__)
if (dim > 7) {
*out = -InnerProductAVX(m, q, dim);
return;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX) {
if (dim > 7) {
*out = -InnerProductAVX(m, q, dim);
return;
}
}
#endif // __AVX__
*out = -InnerProductSSE(m, q, dim);

1
src/ailego/math/inner_product_matrix_int4.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <ailego/internal/cpu_features.h>
#include "distance_matrix_accum_int4.i"
#include "inner_product_matrix.h"

1
src/ailego/math_batch/cosine_distance_batch.h
View File

@ -15,6 +15,7 @@
#pragma once
#include <vector>
#include <ailego/internal/cpu_features.h>
#include <ailego/internal/platform.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/type_helper.h>

40
src/ailego/math_batch/inner_product_distance_batch.h
View File

@ -15,6 +15,7 @@
#pragma once
#include <vector>
#include <ailego/internal/cpu_features.h>
#include <ailego/internal/platform.h>
#include <ailego/utility/math_helper.h>
#include <ailego/utility/type_helper.h>
@ -75,19 +76,25 @@ struct InnerProductDistanceBatchImpl<int8_t, BatchSize> {
// return compute_one_to_many_avx512_int8<ValueType, BatchSize>(
// query, ptrs, prefetch_ptrs, dim, sums);
#if defined(__AVX512VNNI__)
return compute_one_to_many_avx512_vnni_int8<BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
#elif defined(__AVX2__)
return compute_one_to_many_avx2_int8<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
#else
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
return compute_one_to_many_avx512_vnni_int8<BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
}
#endif
#if defined(__AVX2__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
return compute_one_to_many_avx2_int8<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
}
#endif
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
}
static DistanceBatchQueryPreprocessFunc GetQueryPreprocessFunc() {
#if defined(__AVX512VNNI__)
return compute_one_to_many_avx512_vnni_int8_query_preprocess;
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512_VNNI) {
return compute_one_to_many_avx512_vnni_int8_query_preprocess;
}
#endif
return nullptr;
}
@ -107,11 +114,24 @@ struct InnerProductDistanceBatchImpl<ailego::Float16, BatchSize> {
return compute_one_to_many_avx512f_fp16<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
#elif defined(__AVX2__)
return compute_one_to_many_avx_fp16<ValueType, BatchSize>(
return compute_one_to_many_avx2_fp16<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
#else
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
#endif
#if defined(__AVX512F__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX512F) {
return compute_one_to_many_avx512f_fp16<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
}
#endif
#if defined(__AVX2__)
if (zvec::ailego::internal::CpuFeatures::static_flags_.AVX2) {
return compute_one_to_many_avx2_fp16<ValueType, BatchSize>(
query, ptrs, prefetch_ptrs, dim, sums);
}
#endif
return compute_one_to_many_fallback(query, ptrs, prefetch_ptrs, dim, sums);
}
};
@ -148,4 +168,4 @@ struct InnerProductDistanceBatch {
}
};
} // namespace zvec::ailego::DistanceBatch
} // namespace zvec::ailego::DistanceBatch

9
src/ailego/math_batch/inner_product_distance_batch_impl_fp16.h
View File

@ -76,7 +76,9 @@ compute_one_to_many_avx512fp16_fp16(
}
}
#elif defined(__AVX512F__)
#endif
#if defined(__AVX512F__)
template <typename ValueType, size_t dp_batch>
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
@ -164,12 +166,13 @@ compute_one_to_many_avx512f_fp16(
}
}
}
#endif
#elif defined(__AVX2__)
#if defined(__AVX2__)
template <typename ValueType, size_t dp_batch>
static std::enable_if_t<std::is_same_v<ValueType, ailego::Float16>, void>
compute_one_to_many_av2_fp16(
compute_one_to_many_avx2_fp16(
const ailego::Float16 *query, const ailego::Float16 **ptrs,
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
size_t dimensionality, float *results) {

4
src/ailego/math_batch/inner_product_distance_batch_impl_int8.h
View File

@ -21,7 +21,6 @@
namespace zvec::ailego::DistanceBatch {
#if defined(__AVX512VNNI__)
static void compute_one_to_many_avx512_vnni_int8_query_preprocess(void *query,
@ -154,8 +153,9 @@ static void compute_one_to_many_avx512_vnni_int8(
// results[i] = static_cast<float>(temp_results[i]);
// }
// }
#endif
#elif defined(__AVX2__)
#if defined(__AVX2__)
template <typename ValueType, size_t dp_batch>
static std::enable_if_t<std::is_same_v<ValueType, int8_t>, void>

1245
src/ailego/utility/float_helper.cc
View File

File diff suppressed because it is too large Load Diff

108
tests/ailego/internal/cpu_features_test.cc
View File

@ -176,3 +176,111 @@ TEST(CpuFeatures, General) {
EXPECT_TRUE(CpuFeatures::MMX());
#endif
}
TEST(CpuFeatures, Static) {
std::cout << "* F16C: " << CpuFeatures::static_flags_.F16C
<< std::endl;
std::cout << "* SSE: " << CpuFeatures::static_flags_.SSE
<< std::endl;
std::cout << "* SSE2: " << CpuFeatures::static_flags_.SSE2
<< std::endl;
std::cout << "* SSE3: " << CpuFeatures::static_flags_.SSE3
<< std::endl;
std::cout << "* SSSE3: " << CpuFeatures::static_flags_.SSSE3
<< std::endl;
std::cout << "* SSE4_1: " << CpuFeatures::static_flags_.SSE4_1
<< std::endl;
std::cout << "* SSE4_2: " << CpuFeatures::static_flags_.SSE4_2
<< std::endl;
std::cout << "* AVX: " << CpuFeatures::static_flags_.AVX
<< std::endl;
std::cout << "* AVX2: " << CpuFeatures::static_flags_.AVX2
<< std::endl;
std::cout << "* AVX512F: " << CpuFeatures::static_flags_.AVX512F
<< std::endl;
std::cout << "* AVX512DQ: " << CpuFeatures::static_flags_.AVX512DQ
<< std::endl;
std::cout << "* AVX512PF: " << CpuFeatures::static_flags_.AVX512PF
<< std::endl;
std::cout << "* AVX512ER: " << CpuFeatures::static_flags_.AVX512ER
<< std::endl;
std::cout << "* AVX512CD: " << CpuFeatures::static_flags_.AVX512CD
<< std::endl;
std::cout << "* AVX512BW: " << CpuFeatures::static_flags_.AVX512BW
<< std::endl;
std::cout << "* AVX512VL: " << CpuFeatures::static_flags_.AVX512VL
<< std::endl;
std::cout << "* AVX512_IFMA: " << CpuFeatures::static_flags_.AVX512_IFMA
<< std::endl;
std::cout << "* AVX512_VBMI: " << CpuFeatures::static_flags_.AVX512_VBMI
<< std::endl;
std::cout << "* AVX512_VBMI2: " << CpuFeatures::static_flags_.AVX512_VBMI2
<< std::endl;
std::cout << "* AVX512_VNNI: " << CpuFeatures::static_flags_.AVX512_VNNI
<< std::endl;
std::cout << "* AVX512_BITALG: "
<< CpuFeatures::static_flags_.AVX512_BITALG << std::endl;
std::cout << "* AVX512_VPOPCNTDQ: "
<< CpuFeatures::static_flags_.AVX512_VPOPCNTDQ << std::endl;
std::cout << "* AVX512_4VNNIW: "
<< CpuFeatures::static_flags_.AVX512_4VNNIW << std::endl;
std::cout << "* AVX512_4FMAPS: "
<< CpuFeatures::static_flags_.AVX512_4FMAPS << std::endl;
std::cout << "* AVX512_FP16: " << CpuFeatures::static_flags_.AVX512_FP16
<< std::endl;
std::cout << "* CX8: " << CpuFeatures::static_flags_.CX8
<< std::endl;
std::cout << "* CX16: " << CpuFeatures::static_flags_.CX16
<< std::endl;
std::cout << "* PCLMULQDQ: " << CpuFeatures::static_flags_.PCLMULQDQ
<< std::endl;
std::cout << "* VPCLMULQDQ: " << CpuFeatures::static_flags_.VPCLMULQDQ
<< std::endl;
std::cout << "* CMOV: " << CpuFeatures::static_flags_.CMOV
<< std::endl;
std::cout << "* MOVBE: " << CpuFeatures::static_flags_.MOVBE
<< std::endl;
std::cout << "* ERMS: " << CpuFeatures::static_flags_.ERMS
<< std::endl;
std::cout << "* POPCNT: " << CpuFeatures::static_flags_.POPCNT
<< std::endl;
std::cout << "* XSAVE: " << CpuFeatures::static_flags_.XSAVE
<< std::endl;
std::cout << "* FMA: " << CpuFeatures::static_flags_.FMA
<< std::endl;
std::cout << "* ADX: " << CpuFeatures::static_flags_.ADX
<< std::endl;
std::cout << "* GFNI: " << CpuFeatures::static_flags_.GFNI
<< std::endl;
std::cout << "* AES: " << CpuFeatures::static_flags_.AES
<< std::endl;
std::cout << "* VAES: " << CpuFeatures::static_flags_.VAES
<< std::endl;
std::cout << "* RDSEED: " << CpuFeatures::static_flags_.RDSEED
<< std::endl;
std::cout << "* RDRAND: " << CpuFeatures::static_flags_.RDRAND
<< std::endl;
std::cout << "* SHA: " << CpuFeatures::static_flags_.SHA
<< std::endl;
std::cout << "* BMI1: " << CpuFeatures::static_flags_.BMI1
<< std::endl;
std::cout << "* BMI2: " << CpuFeatures::static_flags_.BMI2
<< std::endl;
std::cout << "* CLFLUSH: " << CpuFeatures::static_flags_.CLFLUSH
<< std::endl;
std::cout << "* CLFLUSHOPT: " << CpuFeatures::static_flags_.CLFLUSHOPT
<< std::endl;
std::cout << "* CLWB: " << CpuFeatures::static_flags_.CLWB
<< std::endl;
std::cout << "* RDPID: " << CpuFeatures::static_flags_.RDPID
<< std::endl;
std::cout << "* FPU: " << CpuFeatures::static_flags_.FPU
<< std::endl;
std::cout << "* HT: " << CpuFeatures::static_flags_.HT
<< std::endl;
std::cout << "* VMX: " << CpuFeatures::static_flags_.VMX
<< std::endl;
std::cout << "* HYPERVISOR: " << CpuFeatures::static_flags_.HYPERVISOR
<< std::endl;
}