youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Find solution to fma, change some type define macros

Browse Source
This commit is contained in:
Youmu Liu
2026-06-03 18:06:37 +08:00
parent 9f3fc3b345
commit 6c38bcfade
10 changed files with 444 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.matkernel.h
+14 -13
View File
@@ -12,26 +12,27 @@
#define YMMCOUNT(T) (sizeof(T) / 2)
#define ZMMCOUNT(T) (sizeof(T) / 4)
#define SN_MAT_KERNEL_MEMBER(T) {\
__m128 x[XMMCOUNT(T)]; \
__m256 y[YMMCOUNT(T)]; \
__m512 z[ZMMCOUNT(T)]; \
T m[4][4];\
}
#define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \
typedef union name { \
__m128 f32_x[XMMCOUNT(T)]; \
__m256 f32_y[YMMCOUNT(T)]; \
__m512 f32_z[ZMMCOUNT(T)]; \
\
__m128d f64_x[XMMCOUNT(T)]; \
__m256d f64_y[YMMCOUNT(T)]; \
__m512d f64_z[ZMMCOUNT(T)]; \
T m[4][4]; \
} name
typedef union name SN_MAT_KERNEL_MEMBER(T) name
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t);
#define MAT_KERNEL_DEFINE(T) \
typedef union SN_MAT_KERNEL_TYPE_##T SN_MAT_KERNEL_MEMBER(T) SN_MAT_KERNEL_TYPE_##T
#define SN_MAT_KERNEL_TYPE(T) SN_MAT_KERNEL_TYPE_##T
#define SN_MAT_KERNEL_TYPE_float sn_mat_kernel_float
#define SN_MAT_KERNEL_TYPE_double sn_mat_kernel_double
#define SN_MAT_KERNEL_TYPE_uint64_t sn_mat_kernel_uint64_t
MAT_KERNEL_DEFINE(float);
MAT_KERNEL_DEFINE(double);
MAT_KERNEL_DEFINE(uint64_t);
#define MAT_KERNEL(T) SN_MAT_KERNEL_TYPE(T)
StgSharp.Native/sn-src/m.bufop.x86.c
+152 -150
View File
@@ -1,4 +1,5 @@
#include "sn_target.h"
#include <immintrin.h>
#if SN_IS_ARCH(SN_ARCH_X86_64)
@@ -11,10 +12,10 @@ SN_MK_PROC_DECL_STD(float, sse, , fill)
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f32_x[0] = vec;
ans[i].f32_x[1] = vec;
ans[i].f32_x[2] = vec;
ans[i].f32_x[3] = vec;
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
}
}
@@ -23,8 +24,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fill)
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f32_y[0] = vec;
ans[i].f32_y[1] = vec;
ans[i].y[0] = vec;
ans[i].y[1] = vec;
}
}
@@ -33,23 +34,24 @@ SN_MK_PROC_DECL_STD(float, 512, , fill)
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f32_z[0] = vec;
ans[i].z[0] = vec;
}
}
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
register __m128d vec_d = _mm_set1_pd(((double *)(scalar->data))[0]);
register __m128 vec = _mm_castpd_ps(vec_d);
for (size_t i = 0; i < count; i++) {
ans[i].f64_x[0] = vec;
ans[i].f64_x[1] = vec;
ans[i].f64_x[2] = vec;
ans[i].f64_x[3] = vec;
ans[i].f64_x[4] = vec;
ans[i].f64_x[5] = vec;
ans[i].f64_x[6] = vec;
ans[i].f64_x[7] = vec;
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
ans[i].x[4] = vec;
ans[i].x[5] = vec;
ans[i].x[6] = vec;
ans[i].x[7] = vec;
}
}
@@ -58,10 +60,10 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f64_y[0] = vec;
ans[i].f64_y[1] = vec;
ans[i].f64_y[2] = vec;
ans[i].f64_y[3] = vec;
ans[i].y[0] = _mm256_castpd_ps(vec);
ans[i].y[1] = _mm256_castpd_ps( vec);
ans[i].y[2] = _mm256_castpd_ps( vec);
ans[i].y[3] = _mm256_castpd_ps( vec);
}
}
@@ -70,8 +72,8 @@ SN_MK_PROC_DECL_STD(double, 512, , fill)
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
ans[i].f64_z[1] = vec;
ans[i].z[0] = _mm512_castpd_ps(vec);
ans[i].z[1] = _mm512_castpd_ps(vec);
}
}
@@ -80,15 +82,15 @@ SN_MK_PROC_DECL_STD(float, sse, , add)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
register __m128 c1 = _mm_add_ps(left[i].f32_x[1], right[i].f32_x[1]);
register __m128 c2 = _mm_add_ps(left[i].f32_x[2], right[i].f32_x[2]);
register __m128 c3 = _mm_add_ps(left[i].f32_x[3], right[i].f32_x[3]);
register __m128 c0 = _mm_add_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_add_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_add_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_add_ps(left[i].x[3], right[i].x[3]);
ans[i].f32_x[0] = c0;
ans[i].f32_x[1] = c1;
ans[i].f32_x[2] = c2;
ans[i].f32_x[3] = c3;
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
@@ -98,10 +100,10 @@ SN_MK_PROC_DECL_STD(float, avx, , add)
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].f32_y[0], right[i].f32_y[0]);
register __m256 c1 = _mm256_add_ps(left[i].f32_y[1], right[i].f32_y[1]);
ans[i].f32_y[0] = c0;
ans[i].f32_y[1] = c1;
register __m256 c0 = _mm256_add_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_add_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
@@ -110,8 +112,8 @@ SN_MK_PROC_DECL_STD(float, 512, , add)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
register __m512 c0 = _mm512_add_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(double, sse, , add)
@@ -119,31 +121,31 @@ SN_MK_PROC_DECL_STD(double, sse, , add)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
_mm_castps_pd(right[i].f32_x[0]));
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[1]),
_mm_castps_pd(right[i].f32_x[1]));
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[2]),
_mm_castps_pd(right[i].f32_x[2]));
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[3]),
_mm_castps_pd(right[i].f32_x[3]));
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[4]),
_mm_castps_pd(right[i].f32_x[4]));
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[5]),
_mm_castps_pd(right[i].f32_x[5]));
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[6]),
_mm_castps_pd(right[i].f32_x[6]));
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[7]),
_mm_castps_pd(right[i].f32_x[7]));
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].x[0]),
_mm_castps_pd(right[i].x[0]));
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].x[1]),
_mm_castps_pd(right[i].x[1]));
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].x[2]),
_mm_castps_pd(right[i].x[2]));
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].x[3]),
_mm_castps_pd(right[i].x[3]));
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].x[4]),
_mm_castps_pd(right[i].x[4]));
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].x[5]),
_mm_castps_pd(right[i].x[5]));
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].x[6]),
_mm_castps_pd(right[i].x[6]));
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].x[7]),
_mm_castps_pd(right[i].x[7]));
ans[i].f32_x[0] = _mm_castpd_ps(c0);
ans[i].f32_x[1] = _mm_castpd_ps(c1);
ans[i].f32_x[2] = _mm_castpd_ps(c2);
ans[i].f32_x[3] = _mm_castpd_ps(c3);
ans[i].f32_x[4] = _mm_castpd_ps(c4);
ans[i].f32_x[5] = _mm_castpd_ps(c5);
ans[i].f32_x[6] = _mm_castpd_ps(c6);
ans[i].f32_x[7] = _mm_castpd_ps(c7);
ans[i].x[0] = _mm_castpd_ps(c0);
ans[i].x[1] = _mm_castpd_ps(c1);
ans[i].x[2] = _mm_castpd_ps(c2);
ans[i].x[3] = _mm_castpd_ps(c3);
ans[i].x[4] = _mm_castpd_ps(c4);
ans[i].x[5] = _mm_castpd_ps(c5);
ans[i].x[6] = _mm_castpd_ps(c6);
ans[i].x[7] = _mm_castpd_ps(c7);
}
}
@@ -153,14 +155,14 @@ SN_MK_PROC_DECL_STD(double, avx, , add)
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256d c0 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
register __m256d c1 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
register __m256d c2 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
register __m256d c3 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]);
ans[i].f64_y[0] = c0;
ans[i].f64_y[1] = c1;
ans[i].f64_y[2] = c2;
ans[i].f64_y[3] = c3;
register __m256d c0 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c1 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
register __m256d c2 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c3 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
@@ -169,10 +171,10 @@ SN_MK_PROC_DECL_STD(double, 512, , add)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
register __m512d c1 = _mm512_add_pd(left[i].f64_z[1], right[i].f64_z[1]);
ans[i].f64_z[0] = c0;
ans[i].f64_z[1] = c1;
register __m512d c0 = _mm512_add_pd(_mm512_castps_pd( left[i].z[0]), _mm512_castps_pd( right[i].z[0]));
register __m512d c1 = _mm512_add_pd(_mm512_castps_pd( left[i].z[1]), _mm512_castps_pd( right[i].z[1]));
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
@@ -181,15 +183,15 @@ SN_MK_PROC_DECL_STD(float, sse, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
register __m128 c1 = _mm_sub_ps(left[i].f32_x[1], right[i].f32_x[1]);
register __m128 c2 = _mm_sub_ps(left[i].f32_x[2], right[i].f32_x[2]);
register __m128 c3 = _mm_sub_ps(left[i].f32_x[3], right[i].f32_x[3]);
register __m128 c0 = _mm_sub_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_sub_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_sub_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_sub_ps(left[i].x[3], right[i].x[3]);
ans[i].f32_x[0] = c0;
ans[i].f32_x[1] = c1;
ans[i].f32_x[2] = c2;
ans[i].f32_x[3] = c3;
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
@@ -198,10 +200,10 @@ SN_MK_PROC_DECL_STD(float, avx, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
ans[i].f32_y[0] = c0;
ans[i].f32_y[1] = c1;
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
@@ -210,8 +212,8 @@ SN_MK_PROC_DECL_STD(float, 512, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
@@ -220,23 +222,23 @@ SN_MK_PROC_DECL_STD(double, sse, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
register __m128d c1 = _mm_sub_pd(left[i].f64_x[1], right[i].f64_x[1]);
register __m128d c2 = _mm_sub_pd(left[i].f64_x[2], right[i].f64_x[2]);
register __m128d c3 = _mm_sub_pd(left[i].f64_x[3], right[i].f64_x[3]);
register __m128d c4 = _mm_sub_pd(left[i].f64_x[4], right[i].f64_x[4]);
register __m128d c5 = _mm_sub_pd(left[i].f64_x[5], right[i].f64_x[5]);
register __m128d c6 = _mm_sub_pd(left[i].f64_x[6], right[i].f64_x[6]);
register __m128d c7 = _mm_sub_pd(left[i].f64_x[7], right[i].f64_x[7]);
register __m128d c0 = _mm_sub_pd(left[i].x[0], right[i].x[0]);
register __m128d c1 = _mm_sub_pd(left[i].x[1], right[i].x[1]);
register __m128d c2 = _mm_sub_pd(left[i].x[2], right[i].x[2]);
register __m128d c3 = _mm_sub_pd(left[i].x[3], right[i].x[3]);
register __m128d c4 = _mm_sub_pd(left[i].x[4], right[i].x[4]);
register __m128d c5 = _mm_sub_pd(left[i].x[5], right[i].x[5]);
register __m128d c6 = _mm_sub_pd(left[i].x[6], right[i].x[6]);
register __m128d c7 = _mm_sub_pd(left[i].x[7], right[i].x[7]);
ans[i].f64_x[0] = c0;
ans[i].f64_x[1] = c1;
ans[i].f64_x[2] = c2;
ans[i].f64_x[3] = c3;
ans[i].f64_x[4] = c4;
ans[i].f64_x[5] = c5;
ans[i].f64_x[6] = c6;
ans[i].f64_x[7] = c7;
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
@@ -245,10 +247,10 @@ SN_MK_PROC_DECL_STD(double, avx, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
ans[i].f32_y[0] = c0;
ans[i].f32_y[1] = c1;
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
@@ -257,8 +259,8 @@ SN_MK_PROC_DECL_STD(double, 512, , sub)
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
@@ -268,15 +270,15 @@ SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_mul_ps(right[i].f32_x[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].f32_x[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].f32_x[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].f32_x[3], vec);
register __m128 c0 = _mm_mul_ps(right[i].x[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].x[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].x[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].x[3], vec);
ans[i].f32_x[0] = c0;
ans[i].f32_x[1] = c1;
ans[i].f32_x[2] = c2;
ans[i].f32_x[3] = c3;
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
@@ -286,11 +288,11 @@ SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].f32_y[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].f32_y[1], vec);
register __m256 c0 = _mm256_mul_ps(right[i].y[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].y[1], vec);
ans[i].f32_y[0] = c0;
ans[i].f32_y[1] = c1;
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
@@ -300,9 +302,9 @@ SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].f32_z[0], vec);
register __m512 c0 = _mm512_mul_ps(right[i].z[0], vec);
ans[i].f32_z[0] = c0;
ans[i].z[0] = c0;
}
}
@@ -312,23 +314,23 @@ SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_mul_pd(right[i].f64_x[0], vec);
register __m128d c1 = _mm_mul_pd(right[i].f64_x[1], vec);
register __m128d c2 = _mm_mul_pd(right[i].f64_x[2], vec);
register __m128d c3 = _mm_mul_pd(right[i].f64_x[3], vec);
register __m128d c4 = _mm_mul_pd(right[i].f64_x[4], vec);
register __m128d c5 = _mm_mul_pd(right[i].f64_x[5], vec);
register __m128d c6 = _mm_mul_pd(right[i].f64_x[6], vec);
register __m128d c7 = _mm_mul_pd(right[i].f64_x[7], vec);
register __m128d c0 = _mm_mul_pd(right[i].x[0], vec);
register __m128d c1 = _mm_mul_pd(right[i].x[1], vec);
register __m128d c2 = _mm_mul_pd(right[i].x[2], vec);
register __m128d c3 = _mm_mul_pd(right[i].x[3], vec);
register __m128d c4 = _mm_mul_pd(right[i].x[4], vec);
register __m128d c5 = _mm_mul_pd(right[i].x[5], vec);
register __m128d c6 = _mm_mul_pd(right[i].x[6], vec);
register __m128d c7 = _mm_mul_pd(right[i].x[7], vec);
ans[i].f64_x[0] = c0;
ans[i].f64_x[1] = c1;
ans[i].f64_x[2] = c2;
ans[i].f64_x[3] = c3;
ans[i].f64_x[4] = c4;
ans[i].f64_x[5] = c5;
ans[i].f64_x[6] = c6;
ans[i].f64_x[7] = c7;
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
@@ -338,15 +340,15 @@ SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256d c0 = _mm256_mul_pd(right[i].f64_y[0], vec);
register __m256d c1 = _mm256_mul_pd(right[i].f64_y[1], vec);
register __m256d c2 = _mm256_mul_pd(right[i].f64_y[2], vec);
register __m256d c3 = _mm256_mul_pd(right[i].f64_y[3], vec);
register __m256d c0 = _mm256_mul_pd(right[i].y[0], vec);
register __m256d c1 = _mm256_mul_pd(right[i].y[1], vec);
register __m256d c2 = _mm256_mul_pd(right[i].y[2], vec);
register __m256d c3 = _mm256_mul_pd(right[i].y[3], vec);
ans[i].f64_y[0] = c0;
ans[i].f64_y[1] = c1;
ans[i].f64_y[2] = c2;
ans[i].f64_y[3] = c3;
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
@@ -356,11 +358,11 @@ SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
register __m512d c0 = _mm512_mul_pd(right[i].z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].z[1], vec);
ans[i].f64_z[0] = c0;
ans[i].f64_z[1] = c1;
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
StgSharp.Native/sn-src/m.solve.x86.c
+8 -8
View File
@@ -31,10 +31,10 @@ SN_MK_PROC_DECL_STD(float, sse, , try_plu)
MAT_KERNEL(float) a;
const __m128 z = _mm_setzero_ps();
for (int c = 0; c < 4; ++c) {
__m128 col = _mm_load_ps((float *)&source->f32_x[c]);
_mm_storeu_ps((float *)&a.f32_x[c], col);
_mm_store_ps((float *)&upper->f32_x[c], z);
_mm_store_ps((float *)&lower->f32_x[c], z);
__m128 col = _mm_load_ps((float *)&source->x[c]);
_mm_storeu_ps((float *)&a.x[c], col);
_mm_store_ps((float *)&upper->x[c], z);
_mm_store_ps((float *)&lower->x[c], z);
}
const float eps = 1e-12f;
@@ -121,10 +121,10 @@ SN_MK_PROC_DECL_STD(float, sse, , quality)
register __m128 tiny = _mm_set1_ps(1e-6f);
register __m128 col0 = _mm_load_ps((float *)&right->f32_x[0]);
register __m128 col1 = _mm_load_ps((float *)&right->f32_x[1]);
register __m128 col2 = _mm_load_ps((float *)&right->f32_x[2]);
register __m128 col3 = _mm_load_ps((float *)&right->f32_x[3]);
register __m128 col0 = _mm_load_ps((float *)&right->x[0]);
register __m128 col1 = _mm_load_ps((float *)&right->x[1]);
register __m128 col2 = _mm_load_ps((float *)&right->x[2]);
register __m128 col3 = _mm_load_ps((float *)&right->x[3]);
SN_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
// try in reigister mix later
StgSharp.Native/sn-src/m.transpose.x86.c
+14 -14
View File
@@ -11,26 +11,26 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m128 col0 = right->f32_x[0];
register __m128 col1 = right->f32_x[1];
register __m128 col2 = right->f32_x[2];
register __m128 col3 = right->f32_x[3];
register __m128 col0 = right->x[0];
register __m128 col1 = right->x[1];
register __m128 col2 = right->x[2];
register __m128 col3 = right->x[3];
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
ans->x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->x[3] = _mm_movehl_ps(tmp3, tmp1);
}
SN_MK_PROC_DECL_STD(float, avx, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m256 col_low = right->f32_y[0];
register __m256 col_high = right->f32_y[1];
register __m256 col_low = right->y[0];
register __m256 col_high = right->y[1];
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
@@ -38,19 +38,19 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose)
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
ans->y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
SN_MK_PROC_DECL_STD(float, 512, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m512 matrix = right->f32_z[0];
register __m512 matrix = right->z[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
ans->z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
#endif
StgSharp.Native/sn-src/mk.fma.x86.c
+94 -94
View File
@@ -20,54 +20,54 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->f32_x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]);
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->x[3]);
register __m128 bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_cur->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_cur->f32_x[3]);
const __m128 a0 = _mm_load_ps((float *)&left_cur->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_cur->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_cur->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_cur->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[0]);
bcol = _mm_load_ps((float *)&right_cur->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[1]);
bcol = _mm_load_ps((float *)&right_cur->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[2]);
bcol = _mm_load_ps((float *)&right_cur->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_cur->f32_x[3]);
bcol = _mm_load_ps((float *)&right_cur->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
}
_mm_store_ps((float *)&ans_cur->f32_x[0], c0);
_mm_store_ps((float *)&ans_cur->f32_x[1], c1);
_mm_store_ps((float *)&ans_cur->f32_x[2], c2);
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
_mm_store_ps((float *)&ans_cur->x[0], c0);
_mm_store_ps((float *)&ans_cur->x[1], c1);
_mm_store_ps((float *)&ans_cur->x[2], c2);
_mm_store_ps((float *)&ans_cur->x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
@@ -77,31 +77,31 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
_mm256_store_ps((float *)&ans_cur->y[0], c01);
_mm256_store_ps((float *)&ans_cur->y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
@@ -111,30 +111,30 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
}
_mm256_store_ps((float *)&ans_cur->f32_y[0], c01);
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
_mm256_store_ps((float *)&ans_cur->y[0], c01);
_mm256_store_ps((float *)&ans_cur->y[1], c23);
}
@@ -185,14 +185,14 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128d c0_lo = ans_cur->f64_x[0];
register __m128d c0_hi = ans_cur->f64_x[1];
register __m128d c1_lo = ans_cur->f64_x[2];
register __m128d c1_hi = ans_cur->f64_x[3];
register __m128d c2_lo = ans_cur->f64_x[4];
register __m128d c2_hi = ans_cur->f64_x[5];
register __m128d c3_lo = ans_cur->f64_x[6];
register __m128d c3_hi = ans_cur->f64_x[7];
register __m128d c0_lo = ans_cur->x[0];
register __m128d c0_hi = ans_cur->x[1];
register __m128d c1_lo = ans_cur->x[2];
register __m128d c1_hi = ans_cur->x[3];
register __m128d c2_lo = ans_cur->x[4];
register __m128d c2_hi = ans_cur->x[5];
register __m128d c3_lo = ans_cur->x[6];
register __m128d c3_hi = ans_cur->x[7];
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
@@ -209,14 +209,14 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_MK_FMA_F64_SSE_PHASE(3);
}
ans_cur->f64_x[0] = c0_lo;
ans_cur->f64_x[1] = c0_hi;
ans_cur->f64_x[2] = c1_lo;
ans_cur->f64_x[3] = c1_hi;
ans_cur->f64_x[4] = c2_lo;
ans_cur->f64_x[5] = c2_hi;
ans_cur->f64_x[6] = c3_lo;
ans_cur->f64_x[7] = c3_hi;
ans_cur->x[0] = c0_lo;
ans_cur->x[1] = c0_hi;
ans_cur->x[2] = c1_lo;
ans_cur->x[3] = c1_hi;
ans_cur->x[4] = c2_lo;
ans_cur->x[5] = c2_hi;
ans_cur->x[6] = c3_lo;
ans_cur->x[7] = c3_hi;
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
@@ -225,23 +225,23 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
bcol = _mm256_load_pd((double *)&right_cur->y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
@@ -252,7 +252,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
bcol = _mm256_load_pd((double *)&right_cur->y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
@@ -263,7 +263,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
bcol = _mm256_load_pd((double *)&right_cur->y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
@@ -274,7 +274,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
bcol = _mm256_load_pd((double *)&right_cur->y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
@@ -285,10 +285,10 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
@@ -297,54 +297,54 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]);
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]);
bcol = _mm256_load_pd((double *)&right_cur->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]);
bcol = _mm256_load_pd((double *)&right_cur->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]);
bcol = _mm256_load_pd((double *)&right_cur->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]);
bcol = _mm256_load_pd((double *)&right_cur->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
}
_mm256_store_pd((double *)&ans_cur->f64_y[0], c0);
_mm256_store_pd((double *)&ans_cur->f64_y[1], c1);
_mm256_store_pd((double *)&ans_cur->f64_y[2], c2);
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
@@ -353,33 +353,33 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c01 = _mm512_load_pd((double *)&ans_cur->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans_cur->f64_z[1]);
register __m512d c01 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans_cur->z[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left_cur->z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right_cur->f64_y[0]);
register __m512d b0 = _mm512_broadcast_f64x4(right_cur->y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right_cur->f64_y[1]);
register __m512d b1 = _mm512_broadcast_f64x4(right_cur->y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right_cur->f64_y[2]);
register __m512d b2 = _mm512_broadcast_f64x4(right_cur->y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right_cur->f64_y[3]);
register __m512d b3 = _mm512_broadcast_f64x4(right_cur->y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
}
_mm512_store_pd((double *)&ans_cur->f64_z[0], c01);
_mm512_store_pd((double *)&ans_cur->f64_z[1], c23);
_mm512_store_pd((double *)&ans_cur->z[0], c01);
_mm512_store_pd((double *)&ans_cur->z[1], c23);
}
#endif
StgSharp.Native/sn-src/sk.fma.x86.c
+98 -98
View File
@@ -15,101 +15,101 @@
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__sk_param_std(float);
register __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]);
register __m128 c0 = _mm_load_ps((float *)&ans->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans->x[3]);
register __m128 bcol;
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]);
const __m128 a0 = _mm_load_ps((float *)&left->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right->f32_x[0]);
bcol = _mm_load_ps((float *)&right->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right->f32_x[1]);
bcol = _mm_load_ps((float *)&right->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right->f32_x[2]);
bcol = _mm_load_ps((float *)&right->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right->f32_x[3]);
bcol = _mm_load_ps((float *)&right->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps((float *)&ans->f32_x[0], c0);
_mm_store_ps((float *)&ans->f32_x[1], c1);
_mm_store_ps((float *)&ans->f32_x[2], c2);
_mm_store_ps((float *)&ans->f32_x[3], c3);
_mm_store_ps((float *)&ans->x[0], c0);
_mm_store_ps((float *)&ans->x[1], c1);
_mm_store_ps((float *)&ans->x[2], c2);
_mm_store_ps((float *)&ans->x[3], c3);
}
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
_mm256_store_ps((float *)&ans->f32_y[0], c01);
_mm256_store_ps((float *)&ans->f32_y[1], c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
_mm256_store_ps((float *)&ans->f32_y[0], c01);
_mm256_store_ps((float *)&ans->f32_y[1], c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
SN_SK_PROC_DECL_STD(float, 512, , fma)
@@ -120,16 +120,16 @@ SN_SK_PROC_DECL_STD(float, 512, , fma)
register __m512 b = _mm512_load_ps((float *)right);
// k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left->f32_x[0]);
ak = _mm512_broadcast_f32x4(left->x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left->f32_x[1]);
ak = _mm512_broadcast_f32x4(left->x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left->f32_x[2]);
ak = _mm512_broadcast_f32x4(left->x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left->f32_x[3]);
ak = _mm512_broadcast_f32x4(left->x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
_mm512_store_ps((float *)ans, c);
}
@@ -179,154 +179,154 @@ SN_SK_PROC_DECL_STD(double, sse, , fma)
const double *left_values = &left->m[0][0];
const double *right_values = &right->m[0][0];
register __m128d c0_lo = ans->f64_x[0];
register __m128d c0_hi = ans->f64_x[1];
register __m128d c1_lo = ans->f64_x[2];
register __m128d c1_hi = ans->f64_x[3];
register __m128d c2_lo = ans->f64_x[4];
register __m128d c2_hi = ans->f64_x[5];
register __m128d c3_lo = ans->f64_x[6];
register __m128d c3_hi = ans->f64_x[7];
register __m128d c0_lo = ans->x[0];
register __m128d c0_hi = ans->x[1];
register __m128d c1_lo = ans->x[2];
register __m128d c1_hi = ans->x[3];
register __m128d c2_lo = ans->x[4];
register __m128d c2_hi = ans->x[5];
register __m128d c3_lo = ans->x[6];
register __m128d c3_hi = ans->x[7];
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
ans->f64_x[0] = c0_lo;
ans->f64_x[1] = c0_hi;
ans->f64_x[2] = c1_lo;
ans->f64_x[3] = c1_hi;
ans->f64_x[4] = c2_lo;
ans->f64_x[5] = c2_hi;
ans->f64_x[6] = c3_lo;
ans->f64_x[7] = c3_hi;
ans->x[0] = c0_lo;
ans->x[1] = c0_hi;
ans->x[2] = c1_lo;
ans->x[3] = c1_hi;
ans->x[4] = c2_lo;
ans->x[5] = c2_hi;
ans->x[6] = c3_lo;
ans->x[7] = c3_hi;
}
// clang-format off
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->f64_y[0]);
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->f64_y[1]);
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->f64_y[2]);
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->f64_y[3]);
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->f64_y[0], c0);
_mm256_store_pd((double *)&ans->f64_y[1], c1);
_mm256_store_pd((double *)&ans->f64_y[2], c2);
_mm256_store_pd((double *)&ans->f64_y[3], c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
// clang-format on
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->f64_y[0]);
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->f64_y[1]);
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->f64_y[2]);
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->f64_y[3]);
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->f64_y[0], c0);
_mm256_store_pd((double *)&ans->f64_y[1], c1);
_mm256_store_pd((double *)&ans->f64_y[2], c2);
_mm256_store_pd((double *)&ans->f64_y[3], c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__sk_param_std(double);
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left->z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right->f64_y[0]);
register __m512d b0 = _mm512_broadcast_f64x4(right->y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right->f64_y[1]);
register __m512d b1 = _mm512_broadcast_f64x4(right->y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right->f64_y[2]);
register __m512d b2 = _mm512_broadcast_f64x4(right->y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right->f64_y[3]);
register __m512d b3 = _mm512_broadcast_f64x4(right->y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
_mm512_store_pd((double *)&ans->f64_z[0], c01);
_mm512_store_pd((double *)&ans->f64_z[1], c23);
_mm512_store_pd((double *)&ans->z[0], c01);
_mm512_store_pd((double *)&ans->z[1], c23);
}
#endif
StgSharp.Native/sn-src/vector_dot.c
+16 -16
View File
@@ -6,12 +6,12 @@ static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->f32_x[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], *vector, 0B11110001),
*ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 1));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], *vector, 0B11110001),
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], *vector, 0B11110001),
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
@@ -19,8 +19,8 @@ INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 cons
__m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110010),
__m256 tmp = _mm256_dp_ps(transpose->y[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->y[0], _double, 0B11110010),
0B00100010);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
@@ -29,34 +29,34 @@ INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 cons
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->f32_x[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[0], 0B11110001),
ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[0], 0B11110001),
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[0], 0B11110001),
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
ans[1] = _mm_dp_ps(transpose->f32_x[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[1], 0B11110001),
ans[1] = _mm_dp_ps(transpose->x[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[1], 0B11110001),
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[1], 0B11110001),
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
__m256 temp = transpose->f32_y[0];
__m256 temp = transpose->y[0];
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
temp = transpose->f32_y[1];
temp = transpose->y[1];
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
StgSharp.Native/sn-template/sn_template.fma.x86.h
+47 -40
View File
@@ -2,52 +2,59 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
/*
* requested ZMMs:
*
*/
#define SN_MK_FMA_F32_512_CYCLE(i)\
do{\
\
}while(0)
#define SN_MK_FMA_F32_512_PHASE(i)\
do { \
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
}while(0)
#define SN_MK_FMA_F32_512_PHASE(i) \
do { \
ak = _mm512_broadcast_f32x4(left_##i->x[0]); \
c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \
ak = _mm512_broadcast_f32x4(left_##i->x[0]); \
c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \
ak = _mm512_broadcast_f32x4(left_##i->x[0]); \
c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \
ak = _mm512_broadcast_f32x4(left_##i->x[0]); \
c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \
} while (0)
// double
#define SN_MK_FMA_F64_SSE_PHASE(i) \
do { \
/* k = 0 */ \
#define SN_MK_FMA_F64_SSE_PHASE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 2 */ \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 3 */ \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 2 */ \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 3 */ \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
#endif
#endif