youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add simd id and cpu id

Browse Source
This commit is contained in:
Youmu Liu
2026-01-06 18:04:17 +08:00
parent 88fa653323
commit 3d488abd2e
17 changed files with 372 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/cpuid.x86.c
+14 -10
View File
@@ -56,7 +56,7 @@ static int xgetbv_has_avx512_state(void)
return ((xcr0 & mask) == mask);
}
static int is_intel_glue_avx512(int family, int model)
static int is_intel_dual_avx512(int family, int model)
{
return (family == 6) && (model == 0x55); /* SKX/CLX/CPX */
}
@@ -104,6 +104,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
int has_avx = (ecx & (1 << 28)) != 0;
int has_osxsave = (ecx & (1 << 27)) != 0;
int has_fma = (ecx & (1 << 12)) != 0;
int has_f16c = (ecx & (1 << 29)) != 0;
uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL :
@@ -112,12 +113,15 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx512_bits = 0;
uint64_t amx_bits = 0;
uint64_t avx_bits = 0;
/* Baseline main level */
if (has_sse42)
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) {
avx_bits |= SIMDID_AVX;
/* CPUID.7.0 */
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
const int has_bmi1 = (ebx & (1 << 3)) != 0;
@@ -142,13 +146,13 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
const int has_amx_bf16 = (edx & (1 << 26)) != 0;
if (has_avx2) {
if (has_fma) {
main_lvl = SIMDID_PACK_MAIN((has_bmi1 && has_bmi2) ?
SIMDID_MAIN_LVL_AVX2_FMA_F16C :
SIMDID_MAIN_LVL_AVX2_FMA);
} else {
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
avx_bits |= SIMDID_AVX2;
if (has_fma)
avx_bits |= SIMDID_AVX_FMA;
if (has_f16c)
avx_bits |= SIMDID_AVX_F16C;
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && xgetbv_has_avx512_state()) {
@@ -168,7 +172,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
avx512_bits |= SIMDID_AVX512_VBMI2;
/* impl classification */
const int glue = is_amd || is_intel_glue_avx512(family, model);
const int glue = is_amd || is_intel_dual_avx512(family, model);
const int native512 = is_intel_native_avx512(family, model);
if (glue) {
avx512_bits |= SIMDID_AVX512_IMPL_DUAL;
@@ -194,7 +198,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
}
global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
SIMDID_PACK_AMX(amx_bits);
SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits);
return global_simdid;
}
StgSharp.Native/sn-src/load_intrinsic.x86.c
+27 -19
View File
@@ -13,10 +13,9 @@
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx_fma(sn_intrinsic *context);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID simdid);
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
{
@@ -26,18 +25,21 @@ SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
if (check_isa(AVX512)) {
switch (simdid.make_byte[1]) {
case SIMDID_MAIN_LVL_SSE:
load_intrinsic_sse(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX2:
load_intrinsic_avx(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX512:
load_intrinsic_512(context, simdid);
} else if (check_isa(AVX256)) {
load_intrinsic_avx(context);
} else if (check_isa(SSE)) {
load_intrinsic_sse(context);
break;
}
return simdid;
}
void load_intrinsic_sse(sn_intrinsic *context)
void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
@@ -47,10 +49,10 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose);
context->mat[F32].build_panel = NULL;
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, sse, , fma);
context->mat[F32].panel_fma = KER_PROC(float, sse, , fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = NULL;
@@ -68,8 +70,12 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(sn_intrinsic *context)
void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
{
char avx_seg = id.make_byte[5];
char is_fma = (avx_seg & 1 << 3) != 0;
char is_f16c = (avx_seg & 1 << 4) != 0;
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
@@ -78,10 +84,11 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose);
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, avx, , fma);
context->mat[F32].panel_fma = is_fma ? KER_PROC(float, avx, , fma) :
KER_PROC(float, avx, _fma, fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(avx, float);
@@ -92,7 +99,8 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, avx, , fma);
context->mat[F64].panel_fma = is_fma ? PNL_PROC(double, avx, , fma) :
PNL_PROC(double, avx, _fma, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
@@ -109,10 +117,10 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose);
context->mat[F32].build_panel = BUILD_PANEL(512, float);
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, 512, , fma);
context->mat[F32].panel_fma = KER_PROC(float, 512, , fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(512, float);
StgSharp.Native/sn-src/m.basic.x86.c
+3 -3
View File
@@ -55,7 +55,7 @@ DECLARE_PNL_PROC_ANS(double, 512, , clear)
}
// Matrix transpose implementations
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose)
DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose)
{
register __m128 col0 = right->f32_x[0];
register __m128 col1 = right->f32_x[1];
@@ -72,7 +72,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose)
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
}
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose)
DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose)
{
register __m256 col_low = right->f32_y[0];
register __m256 col_high = right->f32_y[1];
@@ -87,7 +87,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose)
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose)
DECLARE_KER_PROC_RIGHT_ANS(float, 512, ,transpose)
{
register __m512 matrix = right->f32_z[0];
StgSharp.Native/sn-src/m.fma.x86.c
+102 -143
View File
@@ -9,204 +9,163 @@
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
{
const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
const __m128 a2 = PANEL_COL_VEC((*left), 2, 0);
const __m128 a3 = PANEL_COL_VEC((*left), 3, 0);
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
__m128 c0 = PANEL_COL_VEC((*ans), 0, 0);
__m128 c1 = PANEL_COL_VEC((*ans), 1, 0);
__m128 c2 = PANEL_COL_VEC((*ans), 2, 0);
__m128 c3 = PANEL_COL_VEC((*ans), 3, 0);
__m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
__m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
__m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
__m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
__m128 bcol;
/* Column 0 */
bcol = PANEL_COL_VEC((*right), 0, 0);
bcol = _mm_load_ps((float *)&right->f32_x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = PANEL_COL_VEC((*right), 1, 0);
bcol = _mm_load_ps((float *)&right->f32_x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = PANEL_COL_VEC((*right), 2, 0);
bcol = _mm_load_ps((float *)&right->f32_x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = PANEL_COL_VEC((*right), 3, 0);
bcol = _mm_load_ps((float *)&right->f32_x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
PANEL_COL_VEC((*ans), 0, 0) = c0;
PANEL_COL_VEC((*ans), 1, 0) = c1;
PANEL_COL_VEC((*ans), 2, 0) = c2;
PANEL_COL_VEC((*ans), 3, 0) = c3;
ans->f32_x[0] = c0;
ans->f32_x[1] = c1;
ans->f32_x[2] = c2;
ans->f32_x[3] = c3;
}
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol);
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a03, b23_perm, c23);
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a03, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
for (int col = 0; col < 8; ++col) {
register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]);
register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]);
register __m256i idx = _mm256_setzero_si256();
register __m256i increament = _mm256_load_si256(&idx0);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]);
register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]);
register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]);
register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23);
a0 = _mm256_load_ps((float *)&left->vec[4][0]);
a1 = _mm256_load_ps((float *)&left->vec[5][0]);
a2 = _mm256_load_ps((float *)&left->vec[6][0]);
a3 = _mm256_load_ps((float *)&left->vec[7][0]);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx)));
idx = _mm256_add_epi32(idx, increament);
ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx)));
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a03, b23_perm), c23);
_mm256_store_ps((float *)&ans->vec[col][0], ccol);
}
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_add_ps(_mm256_mul_ps(a03, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
const __m512i ones = _mm512_set1_epi32(1);
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
for (int col = 0; col < 16; ++col) {
register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]);
register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]);
register __m512i idx = _mm512_setzero_si512();
register __m512i increament = _mm512_load_si512(&ones);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a, b_perm, c);
register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]);
register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]);
register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]);
register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]);
register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]);
register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]);
register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]);
register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a, b_perm, c);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a, b_perm, c);
a0 = _mm512_load_ps((float *)&left->vec[8][0]);
a1 = _mm512_load_ps((float *)&left->vec[9][0]);
a2 = _mm512_load_ps((float *)&left->vec[10][0]);
a3 = _mm512_load_ps((float *)&left->vec[11][0]);
a4 = _mm512_load_ps((float *)&left->vec[12][0]);
a5 = _mm512_load_ps((float *)&left->vec[13][0]);
a6 = _mm512_load_ps((float *)&left->vec[14][0]);
a7 = _mm512_load_ps((float *)&left->vec[15][0]);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a, b_perm, c);
ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol);
idx = _mm512_add_epi32(idx, increament);
_mm512_store_ps((float *)&ans->vec[col][0], ccol);
}
_mm512_store_ps((float *)ans, c);
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)