From 6c38bcfade5e03a48fb7527ef90fa5517e8a62b2 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 3 Jun 2026 18:06:37 +0800 Subject: [PATCH] Find solution to fma, change some type define macros --- .../Numeric/Runtime/MatrixParallelTask.cs | 2 +- .../sn-head/sn_intrinsic.matkernel.h | 27 +- StgSharp.Native/sn-src/m.bufop.x86.c | 302 +++++++++--------- StgSharp.Native/sn-src/m.solve.x86.c | 16 +- StgSharp.Native/sn-src/m.transpose.x86.c | 28 +- StgSharp.Native/sn-src/mk.fma.x86.c | 188 +++++------ StgSharp.Native/sn-src/sk.fma.x86.c | 196 ++++++------ StgSharp.Native/sn-src/vector_dot.c | 32 +- .../sn-template/sn_template.fma.x86.h | 87 ++--- StgSharpDevelop.sln | 5 - 10 files changed, 444 insertions(+), 439 deletions(-) diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs index de0c78e9..4e20d4aa 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs @@ -47,7 +47,7 @@ namespace StgSharp.Mathematics.Numeric.Runtime internal unsafe ref struct MatrixTask { - [FieldOffset(0)]public L4CacheLine Line; + [FieldOffset(0)]public L4.Handle Line; } diff --git a/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h b/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h index 6e71afc0..fdb38370 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h @@ -12,26 +12,27 @@ #define YMMCOUNT(T) (sizeof(T) / 2) #define ZMMCOUNT(T) (sizeof(T) / 4) +#define SN_MAT_KERNEL_MEMBER(T) {\ + __m128 x[XMMCOUNT(T)]; \ + __m256 y[YMMCOUNT(T)]; \ + __m512 z[ZMMCOUNT(T)]; \ + T m[4][4];\ +} + #define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \ - typedef union name { \ - __m128 f32_x[XMMCOUNT(T)]; \ - __m256 f32_y[YMMCOUNT(T)]; \ - __m512 f32_z[ZMMCOUNT(T)]; \ - \ - __m128d f64_x[XMMCOUNT(T)]; \ - __m256d f64_y[YMMCOUNT(T)]; \ - __m512d f64_z[ZMMCOUNT(T)]; \ - T m[4][4]; \ - } name + typedef union name SN_MAT_KERNEL_MEMBER(T) name SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float); SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double); SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t); +#define MAT_KERNEL_DEFINE(T) \ + typedef union SN_MAT_KERNEL_TYPE_##T SN_MAT_KERNEL_MEMBER(T) SN_MAT_KERNEL_TYPE_##T + #define SN_MAT_KERNEL_TYPE(T) SN_MAT_KERNEL_TYPE_##T -#define SN_MAT_KERNEL_TYPE_float sn_mat_kernel_float -#define SN_MAT_KERNEL_TYPE_double sn_mat_kernel_double -#define SN_MAT_KERNEL_TYPE_uint64_t sn_mat_kernel_uint64_t +MAT_KERNEL_DEFINE(float); +MAT_KERNEL_DEFINE(double); +MAT_KERNEL_DEFINE(uint64_t); #define MAT_KERNEL(T) SN_MAT_KERNEL_TYPE(T) diff --git a/StgSharp.Native/sn-src/m.bufop.x86.c b/StgSharp.Native/sn-src/m.bufop.x86.c index 5a218210..dedb6a78 100644 --- a/StgSharp.Native/sn-src/m.bufop.x86.c +++ b/StgSharp.Native/sn-src/m.bufop.x86.c @@ -1,4 +1,5 @@ #include "sn_target.h" +#include #if SN_IS_ARCH(SN_ARCH_X86_64) @@ -11,10 +12,10 @@ SN_MK_PROC_DECL_STD(float, sse, , fill) __mk_param_std(BUFFER, ANS_SCALAR, float); register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { - ans[i].f32_x[0] = vec; - ans[i].f32_x[1] = vec; - ans[i].f32_x[2] = vec; - ans[i].f32_x[3] = vec; + ans[i].x[0] = vec; + ans[i].x[1] = vec; + ans[i].x[2] = vec; + ans[i].x[3] = vec; } } @@ -23,8 +24,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fill) __mk_param_std(BUFFER, ANS_SCALAR, float); register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); for (size_t i = 0; i < count; i++) { - ans[i].f32_y[0] = vec; - ans[i].f32_y[1] = vec; + ans[i].y[0] = vec; + ans[i].y[1] = vec; } } @@ -33,23 +34,24 @@ SN_MK_PROC_DECL_STD(float, 512, , fill) __mk_param_std(BUFFER, ANS_SCALAR, float); register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { - ans[i].f32_z[0] = vec; + ans[i].z[0] = vec; } } SN_MK_PROC_DECL_STD(double, sse, , fill) { __mk_param_std(BUFFER, ANS_SCALAR, double); - register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]); + register __m128d vec_d = _mm_set1_pd(((double *)(scalar->data))[0]); + register __m128 vec = _mm_castpd_ps(vec_d); for (size_t i = 0; i < count; i++) { - ans[i].f64_x[0] = vec; - ans[i].f64_x[1] = vec; - ans[i].f64_x[2] = vec; - ans[i].f64_x[3] = vec; - ans[i].f64_x[4] = vec; - ans[i].f64_x[5] = vec; - ans[i].f64_x[6] = vec; - ans[i].f64_x[7] = vec; + ans[i].x[0] = vec; + ans[i].x[1] = vec; + ans[i].x[2] = vec; + ans[i].x[3] = vec; + ans[i].x[4] = vec; + ans[i].x[5] = vec; + ans[i].x[6] = vec; + ans[i].x[7] = vec; } } @@ -58,10 +60,10 @@ SN_MK_PROC_DECL_STD(double, avx, , fill) __mk_param_std(BUFFER, ANS_SCALAR, double); register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data))); for (size_t i = 0; i < count; i++) { - ans[i].f64_y[0] = vec; - ans[i].f64_y[1] = vec; - ans[i].f64_y[2] = vec; - ans[i].f64_y[3] = vec; + ans[i].y[0] = _mm256_castpd_ps(vec); + ans[i].y[1] = _mm256_castpd_ps( vec); + ans[i].y[2] = _mm256_castpd_ps( vec); + ans[i].y[3] = _mm256_castpd_ps( vec); } } @@ -70,8 +72,8 @@ SN_MK_PROC_DECL_STD(double, 512, , fill) __mk_param_std(BUFFER, ANS_SCALAR, double); register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]); for (size_t i = 0; i < count; i++) { - ans[i].f64_z[0] = vec; - ans[i].f64_z[1] = vec; + ans[i].z[0] = _mm512_castpd_ps(vec); + ans[i].z[1] = _mm512_castpd_ps(vec); } } @@ -80,15 +82,15 @@ SN_MK_PROC_DECL_STD(float, sse, , add) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { - register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]); - register __m128 c1 = _mm_add_ps(left[i].f32_x[1], right[i].f32_x[1]); - register __m128 c2 = _mm_add_ps(left[i].f32_x[2], right[i].f32_x[2]); - register __m128 c3 = _mm_add_ps(left[i].f32_x[3], right[i].f32_x[3]); + register __m128 c0 = _mm_add_ps(left[i].x[0], right[i].x[0]); + register __m128 c1 = _mm_add_ps(left[i].x[1], right[i].x[1]); + register __m128 c2 = _mm_add_ps(left[i].x[2], right[i].x[2]); + register __m128 c3 = _mm_add_ps(left[i].x[3], right[i].x[3]); - ans[i].f32_x[0] = c0; - ans[i].f32_x[1] = c1; - ans[i].f32_x[2] = c2; - ans[i].f32_x[3] = c3; + ans[i].x[0] = c0; + ans[i].x[1] = c1; + ans[i].x[2] = c2; + ans[i].x[3] = c3; } } @@ -98,10 +100,10 @@ SN_MK_PROC_DECL_STD(float, avx, , add) for (size_t i = 0; i < count; i++) { size_t s = sizeof(MAT_KERNEL(float)); - register __m256 c0 = _mm256_add_ps(left[i].f32_y[0], right[i].f32_y[0]); - register __m256 c1 = _mm256_add_ps(left[i].f32_y[1], right[i].f32_y[1]); - ans[i].f32_y[0] = c0; - ans[i].f32_y[1] = c1; + register __m256 c0 = _mm256_add_ps(left[i].y[0], right[i].y[0]); + register __m256 c1 = _mm256_add_ps(left[i].y[1], right[i].y[1]); + ans[i].y[0] = c0; + ans[i].y[1] = c1; } } @@ -110,8 +112,8 @@ SN_MK_PROC_DECL_STD(float, 512, , add) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { - register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]); - ans[i].f32_z[0] = c0; + register __m512 c0 = _mm512_add_ps(left[i].z[0], right[i].z[0]); + ans[i].z[0] = c0; } } SN_MK_PROC_DECL_STD(double, sse, , add) @@ -119,31 +121,31 @@ SN_MK_PROC_DECL_STD(double, sse, , add) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, double); for (size_t i = 0; i < count; i++) { - register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]), - _mm_castps_pd(right[i].f32_x[0])); - register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[1]), - _mm_castps_pd(right[i].f32_x[1])); - register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[2]), - _mm_castps_pd(right[i].f32_x[2])); - register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[3]), - _mm_castps_pd(right[i].f32_x[3])); - register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[4]), - _mm_castps_pd(right[i].f32_x[4])); - register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[5]), - _mm_castps_pd(right[i].f32_x[5])); - register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[6]), - _mm_castps_pd(right[i].f32_x[6])); - register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[7]), - _mm_castps_pd(right[i].f32_x[7])); + register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].x[0]), + _mm_castps_pd(right[i].x[0])); + register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].x[1]), + _mm_castps_pd(right[i].x[1])); + register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].x[2]), + _mm_castps_pd(right[i].x[2])); + register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].x[3]), + _mm_castps_pd(right[i].x[3])); + register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].x[4]), + _mm_castps_pd(right[i].x[4])); + register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].x[5]), + _mm_castps_pd(right[i].x[5])); + register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].x[6]), + _mm_castps_pd(right[i].x[6])); + register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].x[7]), + _mm_castps_pd(right[i].x[7])); - ans[i].f32_x[0] = _mm_castpd_ps(c0); - ans[i].f32_x[1] = _mm_castpd_ps(c1); - ans[i].f32_x[2] = _mm_castpd_ps(c2); - ans[i].f32_x[3] = _mm_castpd_ps(c3); - ans[i].f32_x[4] = _mm_castpd_ps(c4); - ans[i].f32_x[5] = _mm_castpd_ps(c5); - ans[i].f32_x[6] = _mm_castpd_ps(c6); - ans[i].f32_x[7] = _mm_castpd_ps(c7); + ans[i].x[0] = _mm_castpd_ps(c0); + ans[i].x[1] = _mm_castpd_ps(c1); + ans[i].x[2] = _mm_castpd_ps(c2); + ans[i].x[3] = _mm_castpd_ps(c3); + ans[i].x[4] = _mm_castpd_ps(c4); + ans[i].x[5] = _mm_castpd_ps(c5); + ans[i].x[6] = _mm_castpd_ps(c6); + ans[i].x[7] = _mm_castpd_ps(c7); } } @@ -153,14 +155,14 @@ SN_MK_PROC_DECL_STD(double, avx, , add) for (size_t i = 0; i < count; i++) { size_t s = sizeof(MAT_KERNEL(float)); - register __m256d c0 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]); - register __m256d c1 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]); - register __m256d c2 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]); - register __m256d c3 = _mm256_add_pd(left[i].f64_y[1], right[i].f64_y[1]); - ans[i].f64_y[0] = c0; - ans[i].f64_y[1] = c1; - ans[i].f64_y[2] = c2; - ans[i].f64_y[3] = c3; + register __m256d c0 = _mm256_add_pd(left[i].y[0], right[i].y[0]); + register __m256d c1 = _mm256_add_pd(left[i].y[1], right[i].y[1]); + register __m256d c2 = _mm256_add_pd(left[i].y[0], right[i].y[0]); + register __m256d c3 = _mm256_add_pd(left[i].y[1], right[i].y[1]); + ans[i].y[0] = c0; + ans[i].y[1] = c1; + ans[i].y[2] = c2; + ans[i].y[3] = c3; } } @@ -169,10 +171,10 @@ SN_MK_PROC_DECL_STD(double, 512, , add) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, double); for (size_t i = 0; i < count; i++) { - register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]); - register __m512d c1 = _mm512_add_pd(left[i].f64_z[1], right[i].f64_z[1]); - ans[i].f64_z[0] = c0; - ans[i].f64_z[1] = c1; + register __m512d c0 = _mm512_add_pd(_mm512_castps_pd( left[i].z[0]), _mm512_castps_pd( right[i].z[0])); + register __m512d c1 = _mm512_add_pd(_mm512_castps_pd( left[i].z[1]), _mm512_castps_pd( right[i].z[1])); + ans[i].z[0] = c0; + ans[i].z[1] = c1; } } @@ -181,15 +183,15 @@ SN_MK_PROC_DECL_STD(float, sse, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { - register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]); - register __m128 c1 = _mm_sub_ps(left[i].f32_x[1], right[i].f32_x[1]); - register __m128 c2 = _mm_sub_ps(left[i].f32_x[2], right[i].f32_x[2]); - register __m128 c3 = _mm_sub_ps(left[i].f32_x[3], right[i].f32_x[3]); + register __m128 c0 = _mm_sub_ps(left[i].x[0], right[i].x[0]); + register __m128 c1 = _mm_sub_ps(left[i].x[1], right[i].x[1]); + register __m128 c2 = _mm_sub_ps(left[i].x[2], right[i].x[2]); + register __m128 c3 = _mm_sub_ps(left[i].x[3], right[i].x[3]); - ans[i].f32_x[0] = c0; - ans[i].f32_x[1] = c1; - ans[i].f32_x[2] = c2; - ans[i].f32_x[3] = c3; + ans[i].x[0] = c0; + ans[i].x[1] = c1; + ans[i].x[2] = c2; + ans[i].x[3] = c3; } } @@ -198,10 +200,10 @@ SN_MK_PROC_DECL_STD(float, avx, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { - register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]); - register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]); - ans[i].f32_y[0] = c0; - ans[i].f32_y[1] = c1; + register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]); + register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]); + ans[i].y[0] = c0; + ans[i].y[1] = c1; } } @@ -210,8 +212,8 @@ SN_MK_PROC_DECL_STD(float, 512, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, float); for (size_t i = 0; i < count; i++) { - register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]); - ans[i].f32_z[0] = c0; + register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]); + ans[i].z[0] = c0; } } @@ -220,23 +222,23 @@ SN_MK_PROC_DECL_STD(double, sse, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, double); for (size_t i = 0; i < count; i++) { - register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]); - register __m128d c1 = _mm_sub_pd(left[i].f64_x[1], right[i].f64_x[1]); - register __m128d c2 = _mm_sub_pd(left[i].f64_x[2], right[i].f64_x[2]); - register __m128d c3 = _mm_sub_pd(left[i].f64_x[3], right[i].f64_x[3]); - register __m128d c4 = _mm_sub_pd(left[i].f64_x[4], right[i].f64_x[4]); - register __m128d c5 = _mm_sub_pd(left[i].f64_x[5], right[i].f64_x[5]); - register __m128d c6 = _mm_sub_pd(left[i].f64_x[6], right[i].f64_x[6]); - register __m128d c7 = _mm_sub_pd(left[i].f64_x[7], right[i].f64_x[7]); + register __m128d c0 = _mm_sub_pd(left[i].x[0], right[i].x[0]); + register __m128d c1 = _mm_sub_pd(left[i].x[1], right[i].x[1]); + register __m128d c2 = _mm_sub_pd(left[i].x[2], right[i].x[2]); + register __m128d c3 = _mm_sub_pd(left[i].x[3], right[i].x[3]); + register __m128d c4 = _mm_sub_pd(left[i].x[4], right[i].x[4]); + register __m128d c5 = _mm_sub_pd(left[i].x[5], right[i].x[5]); + register __m128d c6 = _mm_sub_pd(left[i].x[6], right[i].x[6]); + register __m128d c7 = _mm_sub_pd(left[i].x[7], right[i].x[7]); - ans[i].f64_x[0] = c0; - ans[i].f64_x[1] = c1; - ans[i].f64_x[2] = c2; - ans[i].f64_x[3] = c3; - ans[i].f64_x[4] = c4; - ans[i].f64_x[5] = c5; - ans[i].f64_x[6] = c6; - ans[i].f64_x[7] = c7; + ans[i].x[0] = c0; + ans[i].x[1] = c1; + ans[i].x[2] = c2; + ans[i].x[3] = c3; + ans[i].x[4] = c4; + ans[i].x[5] = c5; + ans[i].x[6] = c6; + ans[i].x[7] = c7; } } @@ -245,10 +247,10 @@ SN_MK_PROC_DECL_STD(double, avx, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, double); for (size_t i = 0; i < count; i++) { - register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]); - register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]); - ans[i].f32_y[0] = c0; - ans[i].f32_y[1] = c1; + register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]); + register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]); + ans[i].y[0] = c0; + ans[i].y[1] = c1; } } @@ -257,8 +259,8 @@ SN_MK_PROC_DECL_STD(double, 512, , sub) __mk_param_std(BUFFER, LEFT_RIGHT_ANS, double); for (size_t i = 0; i < count; i++) { - register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]); - ans[i].f32_z[0] = c0; + register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]); + ans[i].z[0] = c0; } } @@ -268,15 +270,15 @@ SN_MK_PROC_DECL_STD(float, sse, , scalar_mul) register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { - register __m128 c0 = _mm_mul_ps(right[i].f32_x[0], vec); - register __m128 c1 = _mm_mul_ps(right[i].f32_x[1], vec); - register __m128 c2 = _mm_mul_ps(right[i].f32_x[2], vec); - register __m128 c3 = _mm_mul_ps(right[i].f32_x[3], vec); + register __m128 c0 = _mm_mul_ps(right[i].x[0], vec); + register __m128 c1 = _mm_mul_ps(right[i].x[1], vec); + register __m128 c2 = _mm_mul_ps(right[i].x[2], vec); + register __m128 c3 = _mm_mul_ps(right[i].x[3], vec); - ans[i].f32_x[0] = c0; - ans[i].f32_x[1] = c1; - ans[i].f32_x[2] = c2; - ans[i].f32_x[3] = c3; + ans[i].x[0] = c0; + ans[i].x[1] = c1; + ans[i].x[2] = c2; + ans[i].x[3] = c3; } } @@ -286,11 +288,11 @@ SN_MK_PROC_DECL_STD(float, avx, , scalar_mul) register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data))); for (size_t i = 0; i < count; i++) { - register __m256 c0 = _mm256_mul_ps(right[i].f32_y[0], vec); - register __m256 c1 = _mm256_mul_ps(right[i].f32_y[1], vec); + register __m256 c0 = _mm256_mul_ps(right[i].y[0], vec); + register __m256 c1 = _mm256_mul_ps(right[i].y[1], vec); - ans[i].f32_y[0] = c0; - ans[i].f32_y[1] = c1; + ans[i].y[0] = c0; + ans[i].y[1] = c1; } } @@ -300,9 +302,9 @@ SN_MK_PROC_DECL_STD(float, 512, , scalar_mul) register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { - register __m512 c0 = _mm512_mul_ps(right[i].f32_z[0], vec); + register __m512 c0 = _mm512_mul_ps(right[i].z[0], vec); - ans[i].f32_z[0] = c0; + ans[i].z[0] = c0; } } @@ -312,23 +314,23 @@ SN_MK_PROC_DECL_STD(double, sse, , scalar_mul) register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]); for (size_t i = 0; i < count; i++) { - register __m128d c0 = _mm_mul_pd(right[i].f64_x[0], vec); - register __m128d c1 = _mm_mul_pd(right[i].f64_x[1], vec); - register __m128d c2 = _mm_mul_pd(right[i].f64_x[2], vec); - register __m128d c3 = _mm_mul_pd(right[i].f64_x[3], vec); - register __m128d c4 = _mm_mul_pd(right[i].f64_x[4], vec); - register __m128d c5 = _mm_mul_pd(right[i].f64_x[5], vec); - register __m128d c6 = _mm_mul_pd(right[i].f64_x[6], vec); - register __m128d c7 = _mm_mul_pd(right[i].f64_x[7], vec); + register __m128d c0 = _mm_mul_pd(right[i].x[0], vec); + register __m128d c1 = _mm_mul_pd(right[i].x[1], vec); + register __m128d c2 = _mm_mul_pd(right[i].x[2], vec); + register __m128d c3 = _mm_mul_pd(right[i].x[3], vec); + register __m128d c4 = _mm_mul_pd(right[i].x[4], vec); + register __m128d c5 = _mm_mul_pd(right[i].x[5], vec); + register __m128d c6 = _mm_mul_pd(right[i].x[6], vec); + register __m128d c7 = _mm_mul_pd(right[i].x[7], vec); - ans[i].f64_x[0] = c0; - ans[i].f64_x[1] = c1; - ans[i].f64_x[2] = c2; - ans[i].f64_x[3] = c3; - ans[i].f64_x[4] = c4; - ans[i].f64_x[5] = c5; - ans[i].f64_x[6] = c6; - ans[i].f64_x[7] = c7; + ans[i].x[0] = c0; + ans[i].x[1] = c1; + ans[i].x[2] = c2; + ans[i].x[3] = c3; + ans[i].x[4] = c4; + ans[i].x[5] = c5; + ans[i].x[6] = c6; + ans[i].x[7] = c7; } } @@ -338,15 +340,15 @@ SN_MK_PROC_DECL_STD(double, avx, , scalar_mul) register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data))); for (size_t i = 0; i < count; i++) { - register __m256d c0 = _mm256_mul_pd(right[i].f64_y[0], vec); - register __m256d c1 = _mm256_mul_pd(right[i].f64_y[1], vec); - register __m256d c2 = _mm256_mul_pd(right[i].f64_y[2], vec); - register __m256d c3 = _mm256_mul_pd(right[i].f64_y[3], vec); + register __m256d c0 = _mm256_mul_pd(right[i].y[0], vec); + register __m256d c1 = _mm256_mul_pd(right[i].y[1], vec); + register __m256d c2 = _mm256_mul_pd(right[i].y[2], vec); + register __m256d c3 = _mm256_mul_pd(right[i].y[3], vec); - ans[i].f64_y[0] = c0; - ans[i].f64_y[1] = c1; - ans[i].f64_y[2] = c2; - ans[i].f64_y[3] = c3; + ans[i].y[0] = c0; + ans[i].y[1] = c1; + ans[i].y[2] = c2; + ans[i].y[3] = c3; } } @@ -356,11 +358,11 @@ SN_MK_PROC_DECL_STD(double, 512, , scalar_mul) register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]); for (size_t i = 0; i < count; i++) { - register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec); - register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec); + register __m512d c0 = _mm512_mul_pd(right[i].z[0], vec); + register __m512d c1 = _mm512_mul_pd(right[i].z[1], vec); - ans[i].f64_z[0] = c0; - ans[i].f64_z[1] = c1; + ans[i].z[0] = c0; + ans[i].z[1] = c1; } } diff --git a/StgSharp.Native/sn-src/m.solve.x86.c b/StgSharp.Native/sn-src/m.solve.x86.c index 5d3126bb..67610959 100644 --- a/StgSharp.Native/sn-src/m.solve.x86.c +++ b/StgSharp.Native/sn-src/m.solve.x86.c @@ -31,10 +31,10 @@ SN_MK_PROC_DECL_STD(float, sse, , try_plu) MAT_KERNEL(float) a; const __m128 z = _mm_setzero_ps(); for (int c = 0; c < 4; ++c) { - __m128 col = _mm_load_ps((float *)&source->f32_x[c]); - _mm_storeu_ps((float *)&a.f32_x[c], col); - _mm_store_ps((float *)&upper->f32_x[c], z); - _mm_store_ps((float *)&lower->f32_x[c], z); + __m128 col = _mm_load_ps((float *)&source->x[c]); + _mm_storeu_ps((float *)&a.x[c], col); + _mm_store_ps((float *)&upper->x[c], z); + _mm_store_ps((float *)&lower->x[c], z); } const float eps = 1e-12f; @@ -121,10 +121,10 @@ SN_MK_PROC_DECL_STD(float, sse, , quality) register __m128 tiny = _mm_set1_ps(1e-6f); - register __m128 col0 = _mm_load_ps((float *)&right->f32_x[0]); - register __m128 col1 = _mm_load_ps((float *)&right->f32_x[1]); - register __m128 col2 = _mm_load_ps((float *)&right->f32_x[2]); - register __m128 col3 = _mm_load_ps((float *)&right->f32_x[3]); + register __m128 col0 = _mm_load_ps((float *)&right->x[0]); + register __m128 col1 = _mm_load_ps((float *)&right->x[1]); + register __m128 col2 = _mm_load_ps((float *)&right->x[2]); + register __m128 col3 = _mm_load_ps((float *)&right->x[3]); SN_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f }; // try in reigister mix later diff --git a/StgSharp.Native/sn-src/m.transpose.x86.c b/StgSharp.Native/sn-src/m.transpose.x86.c index 21cf6c28..a3a2f2d9 100644 --- a/StgSharp.Native/sn-src/m.transpose.x86.c +++ b/StgSharp.Native/sn-src/m.transpose.x86.c @@ -11,26 +11,26 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose) { __mk_param_std(KERNEL, RIGHT_ANS, float); - register __m128 col0 = right->f32_x[0]; - register __m128 col1 = right->f32_x[1]; - register __m128 col2 = right->f32_x[2]; - register __m128 col3 = right->f32_x[3]; + register __m128 col0 = right->x[0]; + register __m128 col1 = right->x[1]; + register __m128 col2 = right->x[2]; + register __m128 col3 = right->x[3]; register __m128 tmp0 = _mm_unpacklo_ps(col0, col1); register __m128 tmp1 = _mm_unpackhi_ps(col0, col1); register __m128 tmp2 = _mm_unpacklo_ps(col2, col3); register __m128 tmp3 = _mm_unpackhi_ps(col2, col3); - ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2); - ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0); - ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3); - ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1); + ans->x[0] = _mm_movelh_ps(tmp0, tmp2); + ans->x[1] = _mm_movehl_ps(tmp2, tmp0); + ans->x[2] = _mm_movelh_ps(tmp1, tmp3); + ans->x[3] = _mm_movehl_ps(tmp3, tmp1); } SN_MK_PROC_DECL_STD(float, avx, , transpose) { __mk_param_std(KERNEL, RIGHT_ANS, float); - register __m256 col_low = right->f32_y[0]; - register __m256 col_high = right->f32_y[1]; + register __m256 col_low = right->y[0]; + register __m256 col_high = right->y[1]; register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high); register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high); @@ -38,19 +38,19 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose) col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20); col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31); - ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)); - ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); + ans->y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)); + ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)); } SN_MK_PROC_DECL_STD(float, 512, , transpose) { __mk_param_std(KERNEL, RIGHT_ANS, float); - register __m512 matrix = right->f32_z[0]; + register __m512 matrix = right->z[0]; const __m512i transpose_indices = _mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix); + ans->z[0] = _mm512_permutexvar_ps(transpose_indices, matrix); } #endif diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index 66ca1e11..dbeb1c38 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -20,54 +20,54 @@ SN_MK_PROC_DECL_STD(float, sse, , fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]); - register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]); - register __m128 c2 = _mm_load_ps((float *)&ans_cur->f32_x[2]); - register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]); + register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]); + register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]); + register __m128 c2 = _mm_load_ps((float *)&ans_cur->x[2]); + register __m128 c3 = _mm_load_ps((float *)&ans_cur->x[3]); register __m128 bcol; for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]); - const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]); - const __m128 a2 = _mm_load_ps((float *)&left_cur->f32_x[2]); - const __m128 a3 = _mm_load_ps((float *)&left_cur->f32_x[3]); + const __m128 a0 = _mm_load_ps((float *)&left_cur->x[0]); + const __m128 a1 = _mm_load_ps((float *)&left_cur->x[1]); + const __m128 a2 = _mm_load_ps((float *)&left_cur->x[2]); + const __m128 a3 = _mm_load_ps((float *)&left_cur->x[3]); /* Column 0 */ - bcol = _mm_load_ps((float *)&right_cur->f32_x[0]); + bcol = _mm_load_ps((float *)&right_cur->x[0]); c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 1 */ - bcol = _mm_load_ps((float *)&right_cur->f32_x[1]); + bcol = _mm_load_ps((float *)&right_cur->x[1]); c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 2 */ - bcol = _mm_load_ps((float *)&right_cur->f32_x[2]); + bcol = _mm_load_ps((float *)&right_cur->x[2]); c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 3 */ - bcol = _mm_load_ps((float *)&right_cur->f32_x[3]); + bcol = _mm_load_ps((float *)&right_cur->x[3]); c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); } - _mm_store_ps((float *)&ans_cur->f32_x[0], c0); - _mm_store_ps((float *)&ans_cur->f32_x[1], c1); - _mm_store_ps((float *)&ans_cur->f32_x[2], c2); - _mm_store_ps((float *)&ans_cur->f32_x[3], c3); + _mm_store_ps((float *)&ans_cur->x[0], c0); + _mm_store_ps((float *)&ans_cur->x[1], c1); + _mm_store_ps((float *)&ans_cur->x[2], c2); + _mm_store_ps((float *)&ans_cur->x[3], c3); } SN_MK_PROC_DECL_STD(float, avx, , fma) @@ -77,31 +77,31 @@ SN_MK_PROC_DECL_STD(float, avx, , fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]); + register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]); for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]); + register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]); - register __m256 a0 = _mm256_broadcast_ps(&left_cur->f32_x[0]); + register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]); c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23); - register __m256 a1 = _mm256_broadcast_ps(&left_cur->f32_x[1]); + register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]); c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23); - register __m256 a2 = _mm256_broadcast_ps(&left_cur->f32_x[2]); + register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]); c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23); - register __m256 a3 = _mm256_broadcast_ps(&left_cur->f32_x[3]); + register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]); c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23); } - _mm256_store_ps((float *)&ans_cur->f32_y[0], c01); - _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); + _mm256_store_ps((float *)&ans_cur->y[0], c01); + _mm256_store_ps((float *)&ans_cur->y[1], c23); } SN_MK_PROC_DECL_STD(float, avx, _fma, fma) @@ -111,30 +111,30 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]); + register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]); for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - register __m256 b01 = _mm256_load_ps((float *)&right_cur->f32_y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right_cur->f32_y[1]); + register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]); - register __m256 a0 = _mm256_broadcast_ps(&left_cur->f32_x[0]); + register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]); c01 = _mm256_fmadd_ps(a0, b01, c01); c23 = _mm256_fmadd_ps(a0, b23, c23); - register __m256 a1 = _mm256_broadcast_ps(&left_cur->f32_x[1]); + register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]); c01 = _mm256_fmadd_ps(a1, b01, c01); c23 = _mm256_fmadd_ps(a1, b23, c23); - register __m256 a2 = _mm256_broadcast_ps(&left_cur->f32_x[2]); + register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]); c01 = _mm256_fmadd_ps(a2, b01, c01); c23 = _mm256_fmadd_ps(a2, b23, c23); - register __m256 a3 = _mm256_broadcast_ps(&left_cur->f32_x[3]); + register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]); c01 = _mm256_fmadd_ps(a3, b01, c01); c23 = _mm256_fmadd_ps(a3, b23, c23); } - _mm256_store_ps((float *)&ans_cur->f32_y[0], c01); - _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); + _mm256_store_ps((float *)&ans_cur->y[0], c01); + _mm256_store_ps((float *)&ans_cur->y[1], c23); } @@ -185,14 +185,14 @@ SN_MK_PROC_DECL_STD(double, sse, , fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - register __m128d c0_lo = ans_cur->f64_x[0]; - register __m128d c0_hi = ans_cur->f64_x[1]; - register __m128d c1_lo = ans_cur->f64_x[2]; - register __m128d c1_hi = ans_cur->f64_x[3]; - register __m128d c2_lo = ans_cur->f64_x[4]; - register __m128d c2_hi = ans_cur->f64_x[5]; - register __m128d c3_lo = ans_cur->f64_x[6]; - register __m128d c3_hi = ans_cur->f64_x[7]; + register __m128d c0_lo = ans_cur->x[0]; + register __m128d c0_hi = ans_cur->x[1]; + register __m128d c1_lo = ans_cur->x[2]; + register __m128d c1_hi = ans_cur->x[3]; + register __m128d c2_lo = ans_cur->x[4]; + register __m128d c2_hi = ans_cur->x[5]; + register __m128d c3_lo = ans_cur->x[6]; + register __m128d c3_hi = ans_cur->x[7]; for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); @@ -209,14 +209,14 @@ SN_MK_PROC_DECL_STD(double, sse, , fma) SN_MK_FMA_F64_SSE_PHASE(3); } - ans_cur->f64_x[0] = c0_lo; - ans_cur->f64_x[1] = c0_hi; - ans_cur->f64_x[2] = c1_lo; - ans_cur->f64_x[3] = c1_hi; - ans_cur->f64_x[4] = c2_lo; - ans_cur->f64_x[5] = c2_hi; - ans_cur->f64_x[6] = c3_lo; - ans_cur->f64_x[7] = c3_hi; + ans_cur->x[0] = c0_lo; + ans_cur->x[1] = c0_hi; + ans_cur->x[2] = c1_lo; + ans_cur->x[3] = c1_hi; + ans_cur->x[4] = c2_lo; + ans_cur->x[5] = c2_hi; + ans_cur->x[6] = c3_lo; + ans_cur->x[7] = c3_hi; } SN_MK_PROC_DECL_STD(double, avx, , fma) @@ -225,23 +225,23 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe - register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]); - register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]); - register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]); - register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]); + register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]); + register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]); + register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]); + register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]); register __m256d bcol; for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]); - const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]); - const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]); - const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]); + const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]); /* Column 0 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]); + bcol = _mm256_load_pd((double *)&right_cur->y[0]); c0 = _mm256_add_pd(c0, _mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)))); c0 = _mm256_add_pd(c0, @@ -252,7 +252,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) _mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)))); /* Column 1 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]); + bcol = _mm256_load_pd((double *)&right_cur->y[1]); c1 = _mm256_add_pd(c1, _mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)))); c1 = _mm256_add_pd(c1, @@ -263,7 +263,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) _mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)))); /* Column 2 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]); + bcol = _mm256_load_pd((double *)&right_cur->y[2]); c2 = _mm256_add_pd(c2, _mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)))); c2 = _mm256_add_pd(c2, @@ -274,7 +274,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) _mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)))); /* Column 3 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]); + bcol = _mm256_load_pd((double *)&right_cur->y[3]); c3 = _mm256_add_pd(c3, _mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)))); c3 = _mm256_add_pd(c3, @@ -285,10 +285,10 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) _mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)))); } - _mm256_store_pd((double *)&ans_cur->f64_y[0], c0); - _mm256_store_pd((double *)&ans_cur->f64_y[1], c1); - _mm256_store_pd((double *)&ans_cur->f64_y[2], c2); - _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); + _mm256_store_pd((double *)&ans_cur->y[0], c0); + _mm256_store_pd((double *)&ans_cur->y[1], c1); + _mm256_store_pd((double *)&ans_cur->y[2], c2); + _mm256_store_pd((double *)&ans_cur->y[3], c3); } SN_MK_PROC_DECL_STD(double, avx, _fma, fma) @@ -297,54 +297,54 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]); - register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]); - register __m256d c2 = _mm256_load_pd((double *)&ans_cur->f64_y[2]); - register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]); + register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]); + register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]); + register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]); + register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]); register __m256d bcol; for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]); - const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]); - const __m256d a2 = _mm256_load_pd((double *)&left_cur->f64_y[2]); - const __m256d a3 = _mm256_load_pd((double *)&left_cur->f64_y[3]); + const __m256d a0 = _mm256_load_pd((double *)&left_cur->y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left_cur->y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left_cur->y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left_cur->y[3]); /* Column 0 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[0]); + bcol = _mm256_load_pd((double *)&right_cur->y[0]); c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0); c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0); c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0); c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0); /* Column 1 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[1]); + bcol = _mm256_load_pd((double *)&right_cur->y[1]); c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1); c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1); c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1); c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1); /* Column 2 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[2]); + bcol = _mm256_load_pd((double *)&right_cur->y[2]); c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2); c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2); c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2); c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2); /* Column 3 */ - bcol = _mm256_load_pd((double *)&right_cur->f64_y[3]); + bcol = _mm256_load_pd((double *)&right_cur->y[3]); c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3); c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3); c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3); c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3); } - _mm256_store_pd((double *)&ans_cur->f64_y[0], c0); - _mm256_store_pd((double *)&ans_cur->f64_y[1], c1); - _mm256_store_pd((double *)&ans_cur->f64_y[2], c2); - _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); + _mm256_store_pd((double *)&ans_cur->y[0], c0); + _mm256_store_pd((double *)&ans_cur->y[1], c1); + _mm256_store_pd((double *)&ans_cur->y[2], c2); + _mm256_store_pd((double *)&ans_cur->y[3], c3); } SN_MK_PROC_DECL_STD(double, 512, , fma) @@ -353,33 +353,33 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); - register __m512d c01 = _mm512_load_pd((double *)&ans_cur->f64_z[0]); - register __m512d c23 = _mm512_load_pd((double *)&ans_cur->f64_z[1]); + register __m512d c01 = _mm512_load_pd((double *)&ans_cur->z[0]); + register __m512d c23 = _mm512_load_pd((double *)&ans_cur->z[1]); for (size_t z = 0; z < common_z; z++) { SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]); - register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]); + register __m512d a01 = _mm512_load_pd((double *)&left_cur->z[0]); + register __m512d a23 = _mm512_load_pd((double *)&left_cur->z[1]); - register __m512d b0 = _mm512_broadcast_f64x4(right_cur->f64_y[0]); + register __m512d b0 = _mm512_broadcast_f64x4(right_cur->y[0]); c01 = _mm512_fmadd_pd(a01, b0, c01); c23 = _mm512_fmadd_pd(a23, b0, c23); - register __m512d b1 = _mm512_broadcast_f64x4(right_cur->f64_y[1]); + register __m512d b1 = _mm512_broadcast_f64x4(right_cur->y[1]); c01 = _mm512_fmadd_pd(a01, b1, c01); c23 = _mm512_fmadd_pd(a23, b1, c23); - register __m512d b2 = _mm512_broadcast_f64x4(right_cur->f64_y[2]); + register __m512d b2 = _mm512_broadcast_f64x4(right_cur->y[2]); c01 = _mm512_fmadd_pd(a01, b2, c01); c23 = _mm512_fmadd_pd(a23, b2, c23); - register __m512d b3 = _mm512_broadcast_f64x4(right_cur->f64_y[3]); + register __m512d b3 = _mm512_broadcast_f64x4(right_cur->y[3]); c01 = _mm512_fmadd_pd(a01, b3, c01); c23 = _mm512_fmadd_pd(a23, b3, c23); } - _mm512_store_pd((double *)&ans_cur->f64_z[0], c01); - _mm512_store_pd((double *)&ans_cur->f64_z[1], c23); + _mm512_store_pd((double *)&ans_cur->z[0], c01); + _mm512_store_pd((double *)&ans_cur->z[1], c23); } #endif diff --git a/StgSharp.Native/sn-src/sk.fma.x86.c b/StgSharp.Native/sn-src/sk.fma.x86.c index 84e4cef5..be8ae792 100644 --- a/StgSharp.Native/sn-src/sk.fma.x86.c +++ b/StgSharp.Native/sn-src/sk.fma.x86.c @@ -15,101 +15,101 @@ SN_SK_PROC_DECL_STD(float, sse, , fma) { __sk_param_std(float); - register __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]); - register __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]); - register __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]); - register __m128 c3 = _mm_load_ps((float *)&ans->f32_x[3]); + register __m128 c0 = _mm_load_ps((float *)&ans->x[0]); + register __m128 c1 = _mm_load_ps((float *)&ans->x[1]); + register __m128 c2 = _mm_load_ps((float *)&ans->x[2]); + register __m128 c3 = _mm_load_ps((float *)&ans->x[3]); register __m128 bcol; - const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]); - const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]); - const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]); - const __m128 a3 = _mm_load_ps((float *)&left->f32_x[3]); + const __m128 a0 = _mm_load_ps((float *)&left->x[0]); + const __m128 a1 = _mm_load_ps((float *)&left->x[1]); + const __m128 a2 = _mm_load_ps((float *)&left->x[2]); + const __m128 a3 = _mm_load_ps((float *)&left->x[3]); /* Column 0 */ - bcol = _mm_load_ps((float *)&right->f32_x[0]); + bcol = _mm_load_ps((float *)&right->x[0]); c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 1 */ - bcol = _mm_load_ps((float *)&right->f32_x[1]); + bcol = _mm_load_ps((float *)&right->x[1]); c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 2 */ - bcol = _mm_load_ps((float *)&right->f32_x[2]); + bcol = _mm_load_ps((float *)&right->x[2]); c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); /* Column 3 */ - bcol = _mm_load_ps((float *)&right->f32_x[3]); + bcol = _mm_load_ps((float *)&right->x[3]); c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); - _mm_store_ps((float *)&ans->f32_x[0], c0); - _mm_store_ps((float *)&ans->f32_x[1], c1); - _mm_store_ps((float *)&ans->f32_x[2], c2); - _mm_store_ps((float *)&ans->f32_x[3], c3); + _mm_store_ps((float *)&ans->x[0], c0); + _mm_store_ps((float *)&ans->x[1], c1); + _mm_store_ps((float *)&ans->x[2], c2); + _mm_store_ps((float *)&ans->x[3], c3); } SN_SK_PROC_DECL_STD(float, avx, , fma) { __sk_param_std(float); - register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); + register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]); - register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); + register __m256 b01 = _mm256_load_ps((float *)&right->y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right->y[1]); - register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]); + register __m256 a0 = _mm256_broadcast_ps(&left->x[0]); c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23); - register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]); + register __m256 a1 = _mm256_broadcast_ps(&left->x[1]); c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23); - register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]); + register __m256 a2 = _mm256_broadcast_ps(&left->x[2]); c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23); - register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]); + register __m256 a3 = _mm256_broadcast_ps(&left->x[3]); c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23); - _mm256_store_ps((float *)&ans->f32_y[0], c01); - _mm256_store_ps((float *)&ans->f32_y[1], c23); + _mm256_store_ps((float *)&ans->y[0], c01); + _mm256_store_ps((float *)&ans->y[1], c23); } SN_SK_PROC_DECL_STD(float, avx, _fma, fma) { __sk_param_std(float); - register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); + register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]); - register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); + register __m256 b01 = _mm256_load_ps((float *)&right->y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right->y[1]); - register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]); + register __m256 a0 = _mm256_broadcast_ps(&left->x[0]); c01 = _mm256_fmadd_ps(a0, b01, c01); c23 = _mm256_fmadd_ps(a0, b23, c23); - register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]); + register __m256 a1 = _mm256_broadcast_ps(&left->x[1]); c01 = _mm256_fmadd_ps(a1, b01, c01); c23 = _mm256_fmadd_ps(a1, b23, c23); - register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]); + register __m256 a2 = _mm256_broadcast_ps(&left->x[2]); c01 = _mm256_fmadd_ps(a2, b01, c01); c23 = _mm256_fmadd_ps(a2, b23, c23); - register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]); + register __m256 a3 = _mm256_broadcast_ps(&left->x[3]); c01 = _mm256_fmadd_ps(a3, b01, c01); c23 = _mm256_fmadd_ps(a3, b23, c23); - _mm256_store_ps((float *)&ans->f32_y[0], c01); - _mm256_store_ps((float *)&ans->f32_y[1], c23); + _mm256_store_ps((float *)&ans->y[0], c01); + _mm256_store_ps((float *)&ans->y[1], c23); } SN_SK_PROC_DECL_STD(float, 512, , fma) @@ -120,16 +120,16 @@ SN_SK_PROC_DECL_STD(float, 512, , fma) register __m512 b = _mm512_load_ps((float *)right); // k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j) register __m512 ak; - ak = _mm512_broadcast_f32x4(left->f32_x[0]); + ak = _mm512_broadcast_f32x4(left->x[0]); c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c); // k=1 - ak = _mm512_broadcast_f32x4(left->f32_x[1]); + ak = _mm512_broadcast_f32x4(left->x[1]); c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c); // k=2 - ak = _mm512_broadcast_f32x4(left->f32_x[2]); + ak = _mm512_broadcast_f32x4(left->x[2]); c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c); // k=3 - ak = _mm512_broadcast_f32x4(left->f32_x[3]); + ak = _mm512_broadcast_f32x4(left->x[3]); c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c); _mm512_store_ps((float *)ans, c); } @@ -179,154 +179,154 @@ SN_SK_PROC_DECL_STD(double, sse, , fma) const double *left_values = &left->m[0][0]; const double *right_values = &right->m[0][0]; - register __m128d c0_lo = ans->f64_x[0]; - register __m128d c0_hi = ans->f64_x[1]; - register __m128d c1_lo = ans->f64_x[2]; - register __m128d c1_hi = ans->f64_x[3]; - register __m128d c2_lo = ans->f64_x[4]; - register __m128d c2_hi = ans->f64_x[5]; - register __m128d c3_lo = ans->f64_x[6]; - register __m128d c3_hi = ans->f64_x[7]; + register __m128d c0_lo = ans->x[0]; + register __m128d c0_hi = ans->x[1]; + register __m128d c1_lo = ans->x[2]; + register __m128d c1_hi = ans->x[3]; + register __m128d c2_lo = ans->x[4]; + register __m128d c2_hi = ans->x[5]; + register __m128d c3_lo = ans->x[6]; + register __m128d c3_hi = ans->x[7]; DOUBLE_KERNEL_FMA_CYCLE(0); DOUBLE_KERNEL_FMA_CYCLE(1); DOUBLE_KERNEL_FMA_CYCLE(2); DOUBLE_KERNEL_FMA_CYCLE(3); - ans->f64_x[0] = c0_lo; - ans->f64_x[1] = c0_hi; - ans->f64_x[2] = c1_lo; - ans->f64_x[3] = c1_hi; - ans->f64_x[4] = c2_lo; - ans->f64_x[5] = c2_hi; - ans->f64_x[6] = c3_lo; - ans->f64_x[7] = c3_hi; + ans->x[0] = c0_lo; + ans->x[1] = c0_hi; + ans->x[2] = c1_lo; + ans->x[3] = c1_hi; + ans->x[4] = c2_lo; + ans->x[5] = c2_hi; + ans->x[6] = c3_lo; + ans->x[7] = c3_hi; } // clang-format off SN_SK_PROC_DECL_STD(double, avx, , fma) { __sk_param_std(double); - register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]); - register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]); - register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]); - register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]); + register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]); + register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]); + register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]); + register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]); register __m256d bcol; - const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); - const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); - const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); - const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]); + const __m256d a0 = _mm256_load_pd((double *)&left->y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left->y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left->y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left->y[3]); /* Column 0 */ - bcol = _mm256_load_pd((double *)&right->f64_y[0]); + bcol = _mm256_load_pd((double *)&right->y[0]); c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0); c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0); c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0); c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0); /* Column 1 */ - bcol = _mm256_load_pd((double *)&right->f64_y[1]); + bcol = _mm256_load_pd((double *)&right->y[1]); c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1); c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1); c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1); c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1); /* Column 2 */ - bcol = _mm256_load_pd((double *)&right->f64_y[2]); + bcol = _mm256_load_pd((double *)&right->y[2]); c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2); c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2); c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2); c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2); /* Column 3 */ - bcol = _mm256_load_pd((double *)&right->f64_y[3]); + bcol = _mm256_load_pd((double *)&right->y[3]); c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3); c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3); c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3); c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3); - _mm256_store_pd((double *)&ans->f64_y[0], c0); - _mm256_store_pd((double *)&ans->f64_y[1], c1); - _mm256_store_pd((double *)&ans->f64_y[2], c2); - _mm256_store_pd((double *)&ans->f64_y[3], c3); + _mm256_store_pd((double *)&ans->y[0], c0); + _mm256_store_pd((double *)&ans->y[1], c1); + _mm256_store_pd((double *)&ans->y[2], c2); + _mm256_store_pd((double *)&ans->y[3], c3); } // clang-format on SN_SK_PROC_DECL_STD(double, avx, _fma, fma) { __sk_param_std(double); - register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]); - register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]); - register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]); - register __m256d c3 = _mm256_load_pd((double *)&ans->f64_y[3]); + register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]); + register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]); + register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]); + register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]); register __m256d bcol; - const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); - const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); - const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); - const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]); + const __m256d a0 = _mm256_load_pd((double *)&left->y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left->y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left->y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left->y[3]); /* Column 0 */ - bcol = _mm256_load_pd((double *)&right->f64_y[0]); + bcol = _mm256_load_pd((double *)&right->y[0]); c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0); c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0); c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0); c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0); /* Column 1 */ - bcol = _mm256_load_pd((double *)&right->f64_y[1]); + bcol = _mm256_load_pd((double *)&right->y[1]); c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1); c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1); c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1); c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1); /* Column 2 */ - bcol = _mm256_load_pd((double *)&right->f64_y[2]); + bcol = _mm256_load_pd((double *)&right->y[2]); c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2); c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2); c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2); c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2); /* Column 3 */ - bcol = _mm256_load_pd((double *)&right->f64_y[3]); + bcol = _mm256_load_pd((double *)&right->y[3]); c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3); c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3); c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3); c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3); - _mm256_store_pd((double *)&ans->f64_y[0], c0); - _mm256_store_pd((double *)&ans->f64_y[1], c1); - _mm256_store_pd((double *)&ans->f64_y[2], c2); - _mm256_store_pd((double *)&ans->f64_y[3], c3); + _mm256_store_pd((double *)&ans->y[0], c0); + _mm256_store_pd((double *)&ans->y[1], c1); + _mm256_store_pd((double *)&ans->y[2], c2); + _mm256_store_pd((double *)&ans->y[3], c3); } SN_SK_PROC_DECL_STD(double, 512, , fma) { __sk_param_std(double); - register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]); - register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]); + register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]); + register __m512d c23 = _mm512_load_pd((double *)&ans->z[1]); - register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]); - register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]); + register __m512d a01 = _mm512_load_pd((double *)&left->z[0]); + register __m512d a23 = _mm512_load_pd((double *)&left->z[1]); - register __m512d b0 = _mm512_broadcast_f64x4(right->f64_y[0]); + register __m512d b0 = _mm512_broadcast_f64x4(right->y[0]); c01 = _mm512_fmadd_pd(a01, b0, c01); c23 = _mm512_fmadd_pd(a23, b0, c23); - register __m512d b1 = _mm512_broadcast_f64x4(right->f64_y[1]); + register __m512d b1 = _mm512_broadcast_f64x4(right->y[1]); c01 = _mm512_fmadd_pd(a01, b1, c01); c23 = _mm512_fmadd_pd(a23, b1, c23); - register __m512d b2 = _mm512_broadcast_f64x4(right->f64_y[2]); + register __m512d b2 = _mm512_broadcast_f64x4(right->y[2]); c01 = _mm512_fmadd_pd(a01, b2, c01); c23 = _mm512_fmadd_pd(a23, b2, c23); - register __m512d b3 = _mm512_broadcast_f64x4(right->f64_y[3]); + register __m512d b3 = _mm512_broadcast_f64x4(right->y[3]); c01 = _mm512_fmadd_pd(a01, b3, c01); c23 = _mm512_fmadd_pd(a23, b3, c23); - _mm512_store_pd((double *)&ans->f64_z[0], c01); - _mm512_store_pd((double *)&ans->f64_z[1], c23); + _mm512_store_pd((double *)&ans->z[0], c01); + _mm512_store_pd((double *)&ans->z[1], c23); } #endif diff --git a/StgSharp.Native/sn-src/vector_dot.c b/StgSharp.Native/sn-src/vector_dot.c index 6b7ca295..d58286ff 100644 --- a/StgSharp.Native/sn-src/vector_dot.c +++ b/StgSharp.Native/sn-src/vector_dot.c @@ -6,12 +6,12 @@ static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 }; INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { - *ans = _mm_dp_ps(transpose->f32_x[0], *vector, 0B11110001); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], *vector, 0B11110001), + *ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001); + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 1)); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], *vector, 0B11110001), + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], *vector, 0B11110001), + *ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], *vector, 0B11110001), SSE_INSERT_SIMPLE(1, 4)); } @@ -19,8 +19,8 @@ INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 cons __m128 *ans) { __m256 _double = _mm256_set_m128(vector[0], vector[0]); - __m256 tmp = _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110001); - tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->f32_y[0], _double, 0B11110010), + __m256 tmp = _mm256_dp_ps(transpose->y[0], _double, 0B11110001); + tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->y[0], _double, 0B11110010), 0B00100010); *ans = _mm256_castps256_ps128( _mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices))); @@ -29,34 +29,34 @@ INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 cons INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { - ans[0] = _mm_dp_ps(transpose->f32_x[0], vector[0], 0B11110001); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[0], 0B11110001), + ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001); + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 2)); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[0], 0B11110001), + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[0], 0B11110001), + ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[0], 0B11110001), SSE_INSERT_SIMPLE(1, 4)); - ans[1] = _mm_dp_ps(transpose->f32_x[0], vector[1], 0B11110001); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[1], vector[1], 0B11110001), + ans[1] = _mm_dp_ps(transpose->x[0], vector[1], 0B11110001); + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 2)); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[2], vector[1], 0B11110001), + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 3)); - ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->f32_x[3], vector[1], 0B11110001), + ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[1], 0B11110001), SSE_INSERT_SIMPLE(1, 4)); } INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) { - __m256 temp = transpose->f32_y[0]; + __m256 temp = transpose->y[0]; __m256 vector_stream = _mm256_set_m128(vector[1], vector[0]); __m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010); - temp = transpose->f32_y[1]; + temp = transpose->y[1]; result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000); diff --git a/StgSharp.Native/sn-template/sn_template.fma.x86.h b/StgSharp.Native/sn-template/sn_template.fma.x86.h index 53918585..87912bc8 100644 --- a/StgSharp.Native/sn-template/sn_template.fma.x86.h +++ b/StgSharp.Native/sn-template/sn_template.fma.x86.h @@ -2,52 +2,59 @@ #if SN_IS_ARCH(SN_ARCH_X86_64) +/* + * requested ZMMs: + * +*/ +#define SN_MK_FMA_F32_512_CYCLE(i)\ +do{\ + \ +}while(0) -#define SN_MK_FMA_F32_512_PHASE(i)\ - do { \ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\ - c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\ - }while(0) - +#define SN_MK_FMA_F32_512_PHASE(i) \ + do { \ + ak = _mm512_broadcast_f32x4(left_##i->x[0]); \ + c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \ + ak = _mm512_broadcast_f32x4(left_##i->x[0]); \ + c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \ + ak = _mm512_broadcast_f32x4(left_##i->x[0]); \ + c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \ + ak = _mm512_broadcast_f32x4(left_##i->x[0]); \ + c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i); \ + } while (0) // double -#define SN_MK_FMA_F64_SSE_PHASE(i) \ - do { \ - /* k = 0 */ \ +#define SN_MK_FMA_F64_SSE_PHASE(i) \ + do { \ + /* k = 0 */ \ register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \ register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \ register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \ register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \ - register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* group = 1 */ \ - a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \ - b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* group = 2 */ \ - a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ - \ - /* group = 3 */ \ - a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \ - a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \ - b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ - c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ - c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 1 */ \ + a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \ + b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 2 */ \ + a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ + \ + /* group = 3 */ \ + a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \ + a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \ + b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \ + c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \ + c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ } while (0) -#endif \ No newline at end of file +#endif diff --git a/StgSharpDevelop.sln b/StgSharpDevelop.sln index a4fa5584..1dcb127e 100644 --- a/StgSharpDevelop.sln +++ b/StgSharpDevelop.sln @@ -83,17 +83,12 @@ Global {0D4D2A8F-2B5B-4F7D-8D63-4A9AF4B4C782}.Release|x64.ActiveCfg = Release|x64 {0D4D2A8F-2B5B-4F7D-8D63-4A9AF4B4C782}.Release|x64.Build.0 = Release|x64 {20CBFF26-6746-402E-992C-7B643E9423EB}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {20CBFF26-6746-402E-992C-7B643E9423EB}.Debug|ARM64.Build.0 = Debug|ARM64 {20CBFF26-6746-402E-992C-7B643E9423EB}.Debug|x64.ActiveCfg = Debug|x64 - {20CBFF26-6746-402E-992C-7B643E9423EB}.Debug|x64.Build.0 = Debug|x64 {20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|ARM64.ActiveCfg = PerformanceDebug|ARM64 {20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|ARM64.Build.0 = PerformanceDebug|ARM64 {20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.ActiveCfg = PerformanceDebug|x64 - {20CBFF26-6746-402E-992C-7B643E9423EB}.PerformanceDebug|x64.Build.0 = PerformanceDebug|x64 {20CBFF26-6746-402E-992C-7B643E9423EB}.Release|ARM64.ActiveCfg = Release|ARM64 - {20CBFF26-6746-402E-992C-7B643E9423EB}.Release|ARM64.Build.0 = Release|ARM64 {20CBFF26-6746-402E-992C-7B643E9423EB}.Release|x64.ActiveCfg = Release|x64 - {20CBFF26-6746-402E-992C-7B643E9423EB}.Release|x64.Build.0 = Release|x64 {10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Debug|ARM64.ActiveCfg = Debug|ARM64 {10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Debug|ARM64.Build.0 = Debug|ARM64 {10B5819F-2A62-41B6-980A-EB9E76A0E95C}.Debug|x64.ActiveCfg = Debug|x64