diff --git a/StgSharp.Native/.clangd b/StgSharp.Native/.clangd new file mode 100644 index 00000000..29788b89 --- /dev/null +++ b/StgSharp.Native/.clangd @@ -0,0 +1,2 @@ +CompileFlags: + CompilationDatabase: . diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 5a8355d3..0fbd62f7 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -4,9 +4,10 @@ #define SSC_INTRIN_HEAD #include "StgSharpNative.h" -#include "sn_intrinsic.matkernel.h" +#include #include #include +#include "sn_intrinsic.matkernel.h" #include #ifdef _MSC_VER diff --git a/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h b/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h index fdb38370..3212b151 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.matkernel.h @@ -22,6 +22,7 @@ #define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \ typedef union name SN_MAT_KERNEL_MEMBER(T) name + SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float); SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double); SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t); diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index c5a0459f..966d44cf 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -133,10 +133,10 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) SN_MK_FMA_F32_AVX2_CYCLE(3); } - SN_FMA_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); - SN_FMA_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); - SN_FMA_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); - SN_FMA_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); + SN_FMA_F32_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); + SN_FMA_F32_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); + SN_FMA_F32_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); + SN_FMA_F32_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); } @@ -169,10 +169,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) SN_MK_FMA_F32_512_CYCLE(3); } - SN_FMA_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); - SN_FMA_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); - SN_FMA_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); - SN_FMA_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); + SN_FMA_F32_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); + SN_FMA_F32_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); + SN_FMA_F32_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); + SN_FMA_F32_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); } @@ -348,36 +348,33 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma) SN_MK_PROC_DECL_STD(double, 512, , fma) { __mk_param_std(TILE, LEFT_RIGHT_ANS, double); - SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); - register __m512d c01 = _mm512_load_pd((double *)&ans_cur->z[0]); - register __m512d c23 = _mm512_load_pd((double *)&ans_cur->z[1]); + register __m512d c0 = _mm512_setzero_pd(); + register __m512d c1 = _mm512_setzero_pd(); + register __m512d c2 = _mm512_setzero_pd(); + register __m512d c3 = _mm512_setzero_pd(); - for (size_t z = 0; z < common_z; z++) { - SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); - SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); + for (size_t z = 0; z < common_z; z+=2) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + z; + SN_MAT_KERNEL_LOCAL const *left_cur = left + z; - register __m512d a01 = _mm512_load_pd((double *)&left_cur->z[0]); - register __m512d a23 = _mm512_load_pd((double *)&left_cur->z[1]); + register __m512d a0 = _mm512_load_pd((double *)&left_cur->z[0]); + register __m512d a1 = _mm512_load_pd((double *)&left_cur->z[1]); + register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]); + register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]); - register __m512d b0 = _mm512_broadcast_f64x4(right_cur->y[0]); - c01 = _mm512_fmadd_pd(a01, b0, c01); - c23 = _mm512_fmadd_pd(a23, b0, c23); - - register __m512d b1 = _mm512_broadcast_f64x4(right_cur->y[1]); - c01 = _mm512_fmadd_pd(a01, b1, c01); - c23 = _mm512_fmadd_pd(a23, b1, c23); - - register __m512d b2 = _mm512_broadcast_f64x4(right_cur->y[2]); - c01 = _mm512_fmadd_pd(a01, b2, c01); - c23 = _mm512_fmadd_pd(a23, b2, c23); - - register __m512d b3 = _mm512_broadcast_f64x4(right_cur->y[3]); - c01 = _mm512_fmadd_pd(a01, b3, c01); - c23 = _mm512_fmadd_pd(a23, b3, c23); + register __m512d b0 = _mm512_load_pd((double *)&left_cur->z[0]); + register __m512d b1 = _mm512_load_pd((double *)&left_cur->z[1]); + register __m512d b2 = _mm512_load_pd((double *)&left_cur->z[2]); + register __m512d b3 = _mm512_load_pd((double *)&left_cur->z[3]); + + SN_MK_FMA_F64_AVX2_CYCLE(0); + SN_MK_FMA_F64_AVX2_CYCLE(1); + SN_MK_FMA_F64_AVX2_CYCLE(2); + SN_MK_FMA_F64_AVX2_CYCLE(3); } - _mm512_store_pd((double *)&ans_cur->z[0], c01); - _mm512_store_pd((double *)&ans_cur->z[1], c23); + _mm512_store_pd((double *)&ans_cur->z[0], c0); + _mm512_store_pd((double *)&ans_cur->z[1], c2); } #endif diff --git a/StgSharp.Native/sn-template/sn_template.fma.x86.h b/StgSharp.Native/sn-template/sn_template.fma.x86.h index 8dfce50a..2002847e 100644 --- a/StgSharp.Native/sn-template/sn_template.fma.x86.h +++ b/StgSharp.Native/sn-template/sn_template.fma.x86.h @@ -18,6 +18,14 @@ c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \ } while (0) +#define SN_MK_FMA_F64_AVX2_CYCLE(i) \ + do { \ + c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \ + c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \ + c2 = _mm512_fmadd_ps(c2, a2, _mm512_permutex_pd(b2, _FMA_SELECT(i))); \ + c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \ + } while (0) + #define SN_FMA_512_PACK_COLUMN(col, target) \ do { \ register __m256 low = _mm512_castps512_ps256(col); \ @@ -29,6 +37,14 @@ _mm_store_ps(target, half128); \ } while (0) +#define SN_FMA_F64_AVX2_PACK_COLUMN(col, target) \ + do { \ + register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \ + register __m128 low128 = _mm256_castps256_ps128(col); \ + register __m128 half128 = _mm_add_ps(high128, low128); \ + _mm_store_ps(target, half128); \ + } while (0) + #define SN_FMA_AVX2_PACK_COLUMN(col, target) \ do { \ register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \