youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better FMA

Browse Source
This commit is contained in:
Youmu Liu
2026-03-05 18:04:34 +08:00
parent dfa6f4ba92
commit 83997f64d0
21 changed files with 893 additions and 1030 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/StgSharp.Native.vcxproj
+2 -2
View File
@@ -176,9 +176,9 @@
<ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\m.bufop.x86.c" />
<ClCompile Include="sn-src\m.solve.x86.c" />
<ClCompile Include="sn-src\m.fma_tile.x86.c" />
<ClCompile Include="sn-src\mk.fma.x86.c" />
<ClCompile Include="sn-src\s.factorial.x86.c" />
<ClCompile Include="sn-src\m.fma.x86.c" />
<ClCompile Include="sn-src\sk.fma.x86.c" />
<ClCompile Include="sn-src\s.hash.x86.c" />
<ClCompile Include="sn-src\load_intrinsic.x86.c" />
<ClCompile Include="sn-src\load_glfw.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
+2 -2
View File
@@ -42,7 +42,7 @@
<ClCompile Include="sn-src\m.basic.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.fma.x86.c">
<ClCompile Include="sn-src\sk.fma.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_intrinsic.x86.c">
@@ -57,7 +57,7 @@
<ClCompile Include="sn-src\simdid.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.fma_tile.x86.c">
<ClCompile Include="sn-src\mk.fma.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\t.affinity.c">
StgSharp.Native/sn-head/sn_intrinsic.std.h
+6 -6
View File
@@ -13,14 +13,14 @@
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL MK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL FORCEINLINE void SN_DECL MK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) const *task)
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL FORCEINLINE void SN_DECL SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
#define _SN_STD__PROBE_0 0
StgSharp.Native/sn-src/m.fma.x86.c
-367
View File
@@ -1,367 +0,0 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// float
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
__m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
__m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
__m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
__m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
__m128 bcol;
/* Column 0 */
bcol = _mm_load_ps((float *)&right->f32_x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right->f32_x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right->f32_x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right->f32_x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps((float *)&ans->f32_x[0], c0);
_mm_store_ps((float *)&ans->f32_x[1], c1);
_mm_store_ps((float *)&ans->f32_x[2], c2);
_mm_store_ps((float *)&ans->f32_x[3], c3);
}
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm0 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm0 = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm0, b_perm0, c);
register __m512 a_perm1 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
register __m512 b_perm1 = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm1, b_perm1, c);
register __m512 a_perm2 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
register __m512 b_perm2 = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm2, b_perm2, c);
register __m512 a_perm3 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
register __m512 b_perm3 = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm3, b_perm3, c);
_mm512_store_ps((float *)ans, c);
}
// double
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
register __m128d c_hi = _mm_load_pd(ans->f64_x + (col * 2 + 1));
/* right matrix column split into two chunks */
register __m128d b_lo = _mm_load_pd(right + (2 * col + 0));
register __m128d b_hi = _mm_load_pd(right + (2 * col + 1));
/* k = 0 */
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0));
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1));
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 1 */
a_lo = _mm_load_pd(left + (1 * 2 + 0));
a_hi = _mm_load_pd(left + (1 * 2 + 1));
b = _mm_shuffle_pd(b_lo, b_lo, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 2 */
a_lo = _mm_load_pd(left + (1 * 2 + 0));
a_hi = _mm_load_pd(left + (1 * 2 + 1));
b = _mm_shuffle_pd(b_hi, b_hi, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 3 */
a_lo = _mm_load_pd(left + (1 * 2 + 0));
a_hi = _mm_load_pd(left + (1 * 2 + 1));
b = _mm_shuffle_pd(b_hi, b_hi, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
ans->f64_x[col * 2 + 0] = c_lo;
ans->f64_x[col * 2 + 1] = c_hi;
}
}
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f64_y[0]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
_mm256_store_pd((double *)&ans->f64_y[0], ccol);
/* Column 1 */
ccol = _mm256_load_pd((double *)&ans->f64_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
_mm256_store_pd((double *)&ans->f64_y[1], ccol);
/* Column 2 */
ccol = _mm256_load_pd((double *)&ans->f64_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
_mm256_store_pd((double *)&ans->f64_y[2], ccol);
/* Column 3 */
ccol = _mm256_load_pd((double *)&ans->f64_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f64_y[0]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
_mm256_store_pd((double *)&ans->f64_y[0], ccol);
/* Column 1 */
ccol = _mm256_load_pd((double *)&ans->f64_y[1]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)), ccol);
_mm256_store_pd((double *)&ans->f64_y[1], ccol);
/* Column 2 */
ccol = _mm256_load_pd((double *)&ans->f64_y[2]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)), ccol);
_mm256_store_pd((double *)&ans->f64_y[2], ccol);
/* Column 3 */
ccol = _mm256_load_pd((double *)&ans->f64_y[3]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)), ccol);
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]);
register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]);
b01 = _mm512_permutexvar_pd(b01_shift, b01);
b23 = _mm512_permutexvar_pd(b23_shift, b23);
register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]);
//group 0
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
//group 2
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
//group 1
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2));
register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2));
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
//group 3
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
_mm512_store_pd((double *)&ans->f64_z[0], c01);
_mm512_store_pd((double *)&ans->f64_z[1], c23);
}
#endif
StgSharp.Native/sn-src/m.fma_tile.x86.c → StgSharp.Native/sn-src/mk.fma.x86.c
+414 -490
View File
@@ -1,490 +1,414 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
#define GET_INDEX(col_len, x, y) (col_len * x + y)
// float
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->f32_x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]);
register __m128 bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_cur->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_cur->f32_x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
}
_mm_store_ps((float *)&ans_cur->f32_x[0], c0);
_mm_store_ps((float *)&ans_cur->f32_x[1], c1);
_mm_store_ps((float *)&ans_cur->f32_x[2], c2);
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256i b01_shift = _mm256_load_si256(&b01_shift_source);
register __m256i b23_shift = _mm256_load_si256(&b23_shift_source);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_add_ps(_mm256_mul_ps(a01, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a23, b23_perm), c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_add_ps(_mm256_mul_ps(a23, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a01, b23_perm), c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
register __m256i b01_shift = _mm256_load_si256(&b01_shift_source);
register __m256i b23_shift = _mm256_load_si256(&b23_shift_source);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
5, 6, 7, 4, //col1
10, 11, 8, 9, //col2
15, 12, 13, 14 //col3
} };
register __m512i b_shift = _mm512_load_si512(&b_shift_source);
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps(ans_cur);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512 a = _mm512_load_ps((float *)left_cur);
register __m512 b = _mm512_load_ps((float *)right_cur);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a_perm, b_perm, c);
}
_mm512_store_ps((float *)ans_cur, c);
}
// double
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1));
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
register __m512i b01_shift = _mm512_load_si512(&b01_shift_source);
register __m512i b23_shift = _mm512_load_si512(&b23_shift_source);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]);
register __m512d b01 = _mm512_load_pd((double *)&right_cur->f64_z[0]);
register __m512d b23 = _mm512_load_pd((double *)&right_cur->f64_z[1]);
b01 = _mm512_permutexvar_pd(b01_shift, b01);
b23 = _mm512_permutexvar_pd(b23_shift, b23);
//group 0
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
//group 2
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
//group 1
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, 0x21);
register __m512d a30 = _mm512_shuffle_f64x2(a01, a23, 0x30);
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
//group 3
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
}
_mm512_store_pd((double *)&ans_cur->f64_z[0], c01);
_mm512_store_pd((double *)&ans_cur->f64_z[1], c23);
}
#endif
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
#define GET_INDEX(col_len, x, y) (col_len * x + y)
// float
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->f32_x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]);
register __m128 bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_cur->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_cur->f32_x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
}
_mm_store_ps((float *)&ans_cur->f32_x[0], c0);
_mm_store_ps((float *)&ans_cur->f32_x[1], c1);
_mm_store_ps((float *)&ans_cur->f32_x[2], c2);
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->f32_x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->f32_x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->f32_x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->f32_x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps((float *)&left_cur->f32_x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps((float *)&left_cur->f32_x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps((float *)&left_cur->f32_x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps((float *)&left_cur->f32_x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps((float *)ans_cur);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right_cur);
// k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left_cur->f32_x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left_cur->f32_x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left_cur->f32_x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left_cur->f32_x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
}
_mm512_store_ps((float *)ans_cur, c);
}
// double
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1));
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
MAT_KERNEL(double)
const *ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z),
const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right_cur->f64_y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right_cur->f64_y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right_cur->f64_y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right_cur->f64_y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
}
_mm512_store_pd((double *)&ans_cur->f64_z[0], c01);
_mm512_store_pd((double *)&ans_cur->f64_z[1], c23);
}
#endif
StgSharp.Native/sn-src/sk.fma.x86.c
+321
View File
@@ -0,0 +1,321 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
#define GET_INDEX(col_len, x, y) (col_len * x + y)
// float
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
register __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
register __m128 bcol;
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right->f32_x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right->f32_x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right->f32_x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right->f32_x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps((float *)&ans->f32_x[0], c0);
_mm_store_ps((float *)&ans->f32_x[1], c1);
_mm_store_ps((float *)&ans->f32_x[2], c2);
_mm_store_ps((float *)&ans->f32_x[3], c3);
}
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
_mm256_store_ps((float *)&ans->f32_y[0], c01);
_mm256_store_ps((float *)&ans->f32_y[1], c23);
}
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps((float *)&left->f32_x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps((float *)&left->f32_x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps((float *)&left->f32_x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps((float *)&left->f32_x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
_mm256_store_ps((float *)&ans->f32_y[0], c01);
_mm256_store_ps((float *)&ans->f32_y[1], c23);
}
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
register __m512 c = _mm512_load_ps((float *)ans);
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right);
// k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left->f32_x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left->f32_x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left->f32_x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left->f32_x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
_mm512_store_ps((float *)ans, c);
}
// double
// clang-format off
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
// clang-format on
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
register __m128d c0_lo = _mm_load_pd(ans->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans->f64_x + (3 * 2 + 1));
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi);
}
// clang-format off
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->f64_y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->f64_y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->f64_y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->f64_y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->f64_y[0], c0);
_mm256_store_pd((double *)&ans->f64_y[1], c1);
_mm256_store_pd((double *)&ans->f64_y[2], c2);
_mm256_store_pd((double *)&ans->f64_y[3], c3);
}
// clang-format on
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->f64_y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->f64_y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->f64_y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->f64_y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->f64_y[0], c0);
_mm256_store_pd((double *)&ans->f64_y[1], c1);
_mm256_store_pd((double *)&ans->f64_y[2], c2);
_mm256_store_pd((double *)&ans->f64_y[3], c3);
}
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right->f64_y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right->f64_y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right->f64_y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right->f64_y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
_mm512_store_pd((double *)&ans->f64_z[0], c01);
_mm512_store_pd((double *)&ans->f64_z[1], c23);
}
#endif