fix: use static array (#222)
* fix: use static array * fix: clang-format * fix: clang-format * fix: clang-format
This commit is contained in:
parent
27c1248b9f
commit
d1e0c7b88e
|
|
@ -28,8 +28,7 @@ compute_one_to_many_inner_product_avx2_fp16(
|
|||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m256, dp_batch> accs;
|
||||
|
||||
__m256 accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_ps();
|
||||
}
|
||||
|
|
@ -42,8 +41,8 @@ compute_one_to_many_inner_product_avx2_fp16(
|
|||
__m256 q1 = _mm256_cvtph_ps(_mm256_castsi256_si128(q));
|
||||
__m256 q2 = _mm256_cvtph_ps(_mm256_extractf128_si256(q, 1));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs_1;
|
||||
std::array<__m256, dp_batch> data_regs_2;
|
||||
__m256 data_regs_1[dp_batch];
|
||||
__m256 data_regs_2[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m256i m =
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptrs[i] + dim));
|
||||
|
|
@ -68,7 +67,7 @@ compute_one_to_many_inner_product_avx2_fp16(
|
|||
__m256 q = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(query + dim)));
|
||||
|
||||
std::array<__m256, dp_batch> data_regs;
|
||||
__m256 data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_cvtph_ps(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(ptrs[i] + dim)));
|
||||
|
|
|
|||
|
|
@ -27,8 +27,7 @@ compute_one_to_many_inner_product_avx512fp16_fp16(
|
|||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m512h, dp_batch> accs;
|
||||
|
||||
__m512h accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm512_setzero_ph();
|
||||
}
|
||||
|
|
@ -37,7 +36,7 @@ compute_one_to_many_inner_product_avx512fp16_fp16(
|
|||
for (; dim + 32 <= dimensionality; dim += 32) {
|
||||
__m512h q = _mm512_loadu_ph(query + dim);
|
||||
|
||||
std::array<__m512h, dp_batch> data_regs;
|
||||
__m512h data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm512_loadu_ph(ptrs[i] + dim);
|
||||
}
|
||||
|
|
@ -83,8 +82,7 @@ compute_one_to_many_inner_product_avx512f_fp16(
|
|||
const ailego::Float16 *query, const ailego::Float16 **ptrs,
|
||||
std::array<const ailego::Float16 *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m512, dp_batch> accs;
|
||||
|
||||
__m512 accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm512_setzero_ps();
|
||||
}
|
||||
|
|
@ -97,8 +95,8 @@ compute_one_to_many_inner_product_avx512f_fp16(
|
|||
__m512 q1 = _mm512_cvtph_ps(_mm512_castsi512_si256(q));
|
||||
__m512 q2 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(q, 1));
|
||||
|
||||
std::array<__m512, dp_batch> data_regs_1;
|
||||
std::array<__m512, dp_batch> data_regs_2;
|
||||
__m512 data_regs_1[dp_batch];
|
||||
__m512 data_regs_2[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m512i m =
|
||||
_mm512_loadu_si512(reinterpret_cast<const __m512i *>(ptrs[i] + dim));
|
||||
|
|
@ -123,7 +121,7 @@ compute_one_to_many_inner_product_avx512f_fp16(
|
|||
__m512 q = _mm512_cvtph_ps(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(query + dim)));
|
||||
|
||||
std::array<__m512, dp_batch> data_regs;
|
||||
__m512 data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm512_cvtph_ps(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(ptrs[i] + dim)));
|
||||
|
|
@ -133,7 +131,7 @@ compute_one_to_many_inner_product_avx512f_fp16(
|
|||
dim += 16;
|
||||
}
|
||||
|
||||
std::array<__m256, dp_batch> acc_new;
|
||||
__m256 acc_new[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
acc_new[i] = _mm256_add_ps(
|
||||
_mm512_castps512_ps256(accs[i]),
|
||||
|
|
|
|||
|
|
@ -39,14 +39,15 @@ compute_one_to_many_inner_product_avx2_fp32(
|
|||
const ValueType *query, const ValueType **ptrs,
|
||||
std::array<const ValueType *, dp_batch> &prefetch_ptrs,
|
||||
size_t dimensionality, float *results) {
|
||||
std::array<__m256, dp_batch> accs;
|
||||
__m256 accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_ps();
|
||||
}
|
||||
size_t dim = 0;
|
||||
for (; dim + 8 <= dimensionality; dim += 8) {
|
||||
__m256 q = _mm256_loadu_ps(query + dim);
|
||||
std::array<__m256, dp_batch> data_regs;
|
||||
|
||||
__m256 data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_loadu_ps(ptrs[i] + dim);
|
||||
}
|
||||
|
|
@ -59,13 +60,15 @@ compute_one_to_many_inner_product_avx2_fp32(
|
|||
accs[i] = _mm256_fnmadd_ps(q, data_regs[i], accs[i]);
|
||||
}
|
||||
}
|
||||
std::array<__m128, dp_batch> sum128_regs;
|
||||
|
||||
__m128 sum128_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
sum128_regs[i] = sum_top_bottom_avx(accs[i]);
|
||||
}
|
||||
if (dim + 4 <= dimensionality) {
|
||||
__m128 q = _mm_loadu_ps(query + dim);
|
||||
std::array<__m128, dp_batch> data_regs;
|
||||
|
||||
__m128 data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm_loadu_ps(ptrs[i] + dim);
|
||||
}
|
||||
|
|
@ -81,7 +84,8 @@ compute_one_to_many_inner_product_avx2_fp32(
|
|||
}
|
||||
if (dim + 2 <= dimensionality) {
|
||||
__m128 q = _mm_setzero_ps();
|
||||
std::array<__m128, dp_batch> data_regs;
|
||||
|
||||
__m128 data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm_setzero_ps();
|
||||
}
|
||||
|
|
@ -95,7 +99,8 @@ compute_one_to_many_inner_product_avx2_fp32(
|
|||
}
|
||||
dim += 2;
|
||||
}
|
||||
std::array<float, dp_batch> res;
|
||||
|
||||
float res[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
res[i] = sum4(sum128_regs[i]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <vector>
|
||||
#include <array>
|
||||
#include <ailego/utility/math_helper.h>
|
||||
#include <zvec/ailego/internal/platform.h>
|
||||
#include <zvec/ailego/utility/type_helper.h>
|
||||
|
|
@ -27,14 +27,15 @@ compute_one_to_many_inner_product_avx2_int8(
|
|||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, dp_batch> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results) {
|
||||
std::vector<__m256i> accs(dp_batch);
|
||||
__m256i accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm256_setzero_si256();
|
||||
}
|
||||
size_t dim = 0;
|
||||
for (; dim + 32 <= dimensionality; dim += 32) {
|
||||
__m256i q = _mm256_loadu_si256((const __m256i *)(query + dim));
|
||||
std::vector<__m256i> data_regs(dp_batch);
|
||||
|
||||
__m256i data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] = _mm256_loadu_si256((const __m256i *)(ptrs[i] + dim));
|
||||
}
|
||||
|
|
@ -63,7 +64,8 @@ compute_one_to_many_inner_product_avx2_int8(
|
|||
_mm256_add_epi32(accs[i], _mm256_add_epi32(prod_lo[i], prod_hi[i]));
|
||||
}
|
||||
}
|
||||
std::array<int, dp_batch> temp_results;
|
||||
|
||||
int temp_results[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
__m128i lo = _mm256_castsi256_si128(accs[i]);
|
||||
__m128i hi = _mm256_extracti128_si256(accs[i], 1);
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ static void compute_one_to_many_inner_product_avx512_vnni_int8(
|
|||
const int8_t *query, const int8_t **ptrs,
|
||||
std::array<const int8_t *, dp_batch> &prefetch_ptrs, size_t dimensionality,
|
||||
float *results) {
|
||||
std::array<__m512i, dp_batch> accs;
|
||||
__m512i accs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
accs[i] = _mm512_setzero_si512();
|
||||
}
|
||||
|
|
@ -60,7 +60,8 @@ static void compute_one_to_many_inner_product_avx512_vnni_int8(
|
|||
for (; dim + 64 <= dimensionality; dim += 64) {
|
||||
__m512i q =
|
||||
_mm512_loadu_si512(reinterpret_cast<const __m512i *>(query + dim));
|
||||
std::array<__m512i, dp_batch> data_regs;
|
||||
|
||||
__m512i data_regs[dp_batch];
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
data_regs[i] =
|
||||
_mm512_loadu_si512(reinterpret_cast<const __m512i *>(ptrs[i] + dim));
|
||||
|
|
@ -74,7 +75,8 @@ static void compute_one_to_many_inner_product_avx512_vnni_int8(
|
|||
accs[i] = _mm512_dpbusd_epi32(accs[i], q, data_regs[i]);
|
||||
}
|
||||
}
|
||||
std::array<int, dp_batch> temp_results{};
|
||||
|
||||
int temp_results[dp_batch]{};
|
||||
for (size_t i = 0; i < dp_batch; ++i) {
|
||||
temp_results[i] = _mm512_reduce_add_epi32(accs[i]);
|
||||
}
|
||||
|
|
@ -88,6 +90,7 @@ static void compute_one_to_many_inner_product_avx512_vnni_int8(
|
|||
results[i] = static_cast<float>(temp_results[i]);
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// #elif defined(__AVX512BW__)
|
||||
// // TODO: this version is problematic
|
||||
|
|
|
|||
Loading…
Reference in New Issue