diff --git a/StgSharp.Native/sn-head/sn_intrincontext.x86.h b/StgSharp.Native/sn-head/sn_intrincontext.x86.h index 8bbd39a4..b68e0e63 100644 --- a/StgSharp.Native/sn-head/sn_intrincontext.x86.h +++ b/StgSharp.Native/sn-head/sn_intrincontext.x86.h @@ -154,10 +154,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma); DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma); DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma); -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma); +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma); #pragma endregion diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index ee4edb7a..b17ec566 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -63,7 +63,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_transpose = NULL; context->mat[F64].build_panel = NULL; context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear); - context->mat[F64].panel_fma = PNL_PROC(double, sse, , fma); + context->mat[F64].panel_fma = KER_PROC(double, sse, , fma); context->mat[F64].pivot = NULL; context->mat[F64].store_panel = NULL; @@ -99,8 +99,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_transpose = NULL; context->mat[F64].build_panel = NULL; context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear); - context->mat[F64].panel_fma = is_fma ? PNL_PROC(double, avx, , fma) : - PNL_PROC(double, avx, _fma, fma); + context->mat[F64].panel_fma = is_fma ? KER_PROC(double, avx, , fma) : + KER_PROC(double, avx, _fma, fma); context->mat[F64].pivot = NULL; context->mat[F64].store_panel = NULL; @@ -131,7 +131,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_transpose = NULL; context->mat[F64].build_panel = BUILD_PANEL(512, double); context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear); - context->mat[F64].panel_fma = PNL_PROC(double, 512, , fma); + context->mat[F64].panel_fma = KER_PROC(double, 512, , fma); context->mat[F64].pivot = NULL; context->mat[F64].store_panel = STORE_PANEL(512, double); diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c index b0fc9f20..4a931a8e 100644 --- a/StgSharp.Native/sn-src/m.fma.x86.c +++ b/StgSharp.Native/sn-src/m.fma.x86.c @@ -59,8 +59,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) { - static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 }; - static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 }; + static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; + static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); @@ -105,8 +105,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) { - static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 }; - static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 }; + static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; + static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); @@ -151,7 +151,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) { - static const __m512i b_shift = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, 12, 13, 14 }; + static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, + 12, 13, 14 } }; register __m512 a = _mm512_load_ps((float *)left); register __m512 b = _mm512_load_ps((float *)right); @@ -267,55 +268,93 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) ans->f64_y[0] = ccol; } -DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma) +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma) +{ + static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; + static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; + + register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]); + register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]); + + register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]); + register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]); + b01 = _mm512_permutexvar_pd(b01_shift, b01); + b23 = _mm512_permutexvar_pd(b23_shift, b23); + + register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]); + register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]); + + //group 0 + register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0)); + register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0)); + c01 = _mm512_fmadd_pd(a01, b01_perm, c01); + c23 = _mm512_fmadd_pd(a23, b23_perm, c23); + + //group 2 + b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2)); + b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2)); + c01 = _mm512_fmadd_pd(a23, b01_perm, c01); + c23 = _mm512_fmadd_pd(a01, b23_perm, c23); + + //group 1 + register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2)); + register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2)); + b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1)); + b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1)); + c01 = _mm512_fmadd_pd(a12, b01_perm, c01); + c23 = _mm512_fmadd_pd(a30, b23_perm, c23); + + //group 3 + b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3)); + b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3)); + c01 = _mm512_fmadd_pd(a30, b01_perm, c01); + c23 = _mm512_fmadd_pd(a12, b23_perm, c23); + + ans->f64_z[0] = c01; + ans->f64_z[1] = c23; +} + +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma) { for (int col = 0; col < 4; ++col) { /* current answer column (two chunks: rows 0-1 and 2-3) */ - register __m128d c_lo = _mm_castps_pd(PANEL_COL_VEC((*ans), col, 0)); - register __m128d c_hi = _mm_castps_pd(PANEL_COL_VEC((*ans), col, 1)); + register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0)); + register __m128d c_hi = _mm_load_pd(ans->f64_x + (col * 2 + 1)); /* right matrix column split into two chunks */ - register __m128d b_lo = _mm_castps_pd(PANEL_COL_VEC((*right), col, 0)); - register __m128d b_hi = _mm_castps_pd(PANEL_COL_VEC((*right), col, 1)); + register __m128d b_lo = _mm_load_pd(right + (2 * col + 0)); + register __m128d b_hi = _mm_load_pd(right + (2 * col + 1)); /* k = 0 */ - { - register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 0, 0)); - register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 0, 1)); - register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); - c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); - c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); - } + register __m128d a_lo = _mm_load_pd(left + (0 * 2, 0)); + register __m128d a_hi = _mm_load_pd(left + (0 * 2, 1)); + register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); + c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); + c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); /* k = 1 */ - { - register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 1, 0)); - register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 1, 1)); - register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x3); - c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); - c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); - } + a_lo = _mm_load_pd(left + (1 * 2, 0)); + a_hi = _mm_load_pd(left + (1 * 2, 1)); + b = _mm_shuffle_pd(b_lo, b_lo, 0x3); + c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); + c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); /* k = 2 */ - { - register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 2, 0)); - register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 2, 1)); - register __m128d b = _mm_shuffle_pd(b_hi, b_hi, 0x0); - c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); - c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); - } + a_lo = _mm_load_pd(left + (1 * 2, 0)); + a_hi = _mm_load_pd(left + (1 * 2, 1)); + b = _mm_shuffle_pd(b_hi, b_hi, 0x0); + c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); + c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); /* k = 3 */ - { - register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 3, 0)); - register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 3, 1)); - register __m128d b = _mm_shuffle_pd(b_hi, b_hi, 0x3); - c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); - c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); - } + a_lo = _mm_load_pd(left + (1 * 2, 0)); + a_hi = _mm_load_pd(left + (1 * 2, 1)); + b = _mm_shuffle_pd(b_hi, b_hi, 0x3); + c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b)); + c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b)); - PANEL_COL_VEC((*ans), col, 0) = _mm_castpd_ps(c_lo); - PANEL_COL_VEC((*ans), col, 1) = _mm_castpd_ps(c_hi); + ans->f64_x[col * 2 + 0] = c_lo; + ans->f64_x[col * 2 + 1] = c_hi; } }