diff --git a/StgSharp.Native/sn-head/StgSharpNative.h b/StgSharp.Native/sn-head/StgSharpNative.h index 0f141334..f8d3662a 100644 --- a/StgSharp.Native/sn-head/StgSharpNative.h +++ b/StgSharp.Native/sn-head/StgSharpNative.h @@ -1,10 +1,10 @@ -#ifdef _MSC_VER -#pragma once -#endif - #ifndef SNative #define SNative +#ifdef __cplusplus +#error "This header is not meant to be included in C++ code. StgSharp.Native is a C library and should be used with C linkage." +#endif + #include "sn_internal.h" #include "sn_target.h" @@ -26,10 +26,6 @@ #ifndef SN_ #endif -#define ALIGN(simdVec) _mm_loadu_ps(&simdVec) -#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec) -#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec) - typedef struct Image { int width; int height; diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 0fbd62f7..dafeb3a2 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -4,6 +4,7 @@ #define SSC_INTRIN_HEAD #include "StgSharpNative.h" +#include "sn_target.h" #include #include #include @@ -26,9 +27,31 @@ #define M256_PERMUTE(s3, s2, s1, s0) \ ((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0))) #define BIT_CVRT(pack, index, T) *(T *)(pack->data + index) + +#if SN_IS_ARCH(x86_64) + +#define vec_128 __m128 +#define vec_256 __m256 +#define vec_512 __m512 + #define zero_vec _mm_setzero_ps() #define zero_vec_256 _mm256_setzero_ps() +#define ALIGN(simdVec) _mm_loadu_ps(&simdVec) +#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec) +#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec) + +#else + +#define zero_vec +#define zero_vec_256 + +#define ALIGN(simdVec) +#define ALIGN256(simdVec) +#define ALIGN512(simdVec) + +#endif + #define SPAN(T) \ struct SPAN_T { \ T *pointer; \ @@ -126,6 +149,4 @@ typedef struct sn_intrinsic { typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask; -#include "sn_target.h" - #endif diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index 966d44cf..f399e746 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -71,61 +71,27 @@ SN_MK_PROC_DECL_STD(float, sse, , fma) } SN_MK_PROC_DECL_STD(float, avx, , fma) -{ - __mk_param_std(TILE, LEFT_RIGHT_ANS, float); - - SN_MAT_KERNEL_LOCAL *restrict ans_cur = - ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - - register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]); - - for (size_t z = 0; z < common_z; z++) { - SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); - SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - - register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]); - - register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]); - c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01); - c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23); - register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]); - c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01); - c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23); - register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]); - c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01); - c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23); - register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]); - c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01); - c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23); - } - _mm256_store_ps((float *)&ans_cur->y[0], c01); - _mm256_store_ps((float *)&ans_cur->y[1], c23); -} - -SN_MK_PROC_DECL_STD(float, avx, _fma, fma) { __mk_param_std(TILE, LEFT_RIGHT_ANS, float); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); - register __m256 c0 = _mm256_setzero_ps(); - register __m256 c1 = _mm256_setzero_ps(); - register __m256 c2 = _mm256_setzero_ps(); - register __m256 c3 = _mm256_setzero_ps(); + register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]); + register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]); + register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]); + register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]); for (size_t z = 0; z < common_z; z+=2) { const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z; - register __m256 b0 = _mm256_load_ps((float *)&b_base[0].z[0]); - register __m256 b1 = _mm256_load_ps((float *)&b_base[1].z[0]); - register __m256 b2 = _mm256_load_ps((float *)&b_base[2].z[0]); - register __m256 b3 = _mm256_load_ps((float *)&b_base[3].z[0]); + register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]); + register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]); + register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]); + register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]); - register __m256 a0 = _mm256_load_ps((float *)&a_base[0].z[0]); - register __m256 a1 = _mm256_load_ps((float *)&a_base[1].z[0]); - register __m256 a2 = _mm256_load_ps((float *)&a_base[2].z[0]); - register __m256 a3 = _mm256_load_ps((float *)&a_base[3].z[0]); + register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]); + register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]); + register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]); + register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]); SN_MK_FMA_F32_AVX2_CYCLE(0); SN_MK_FMA_F32_AVX2_CYCLE(1); @@ -133,10 +99,45 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) SN_MK_FMA_F32_AVX2_CYCLE(3); } - SN_FMA_F32_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); - SN_FMA_F32_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); - SN_FMA_F32_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); - SN_FMA_F32_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); + _mm256_store_ps( (float*)&(ans_cur->y[0]),c0); + _mm256_store_ps( (float*)&(ans_cur->y[1]),c1); + _mm256_store_ps( (float*)&(ans_cur->y[2]),c2); + _mm256_store_ps( (float*)&(ans_cur->y[3]),c3); +} + +SN_MK_PROC_DECL_STD(float, avx, _fma, fma) +{ + __mk_param_std(TILE, LEFT_RIGHT_ANS, float); + SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); + register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]); + register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]); + register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]); + register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]); + + for (size_t z = 0; z < common_z; z+=2) { + const SN_MAT_KERNEL_LOCAL *a_base = left + z; + const SN_MAT_KERNEL_LOCAL *b_base = right + z; + + register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]); + register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]); + register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]); + register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]); + + register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]); + register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]); + register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]); + register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]); + + SN_MK_FMA_F32_AVX2_FMA_CYCLE(0); + SN_MK_FMA_F32_AVX2_FMA_CYCLE(1); + SN_MK_FMA_F32_AVX2_FMA_CYCLE(2); + SN_MK_FMA_F32_AVX2_FMA_CYCLE(3); + + } + _mm256_store_ps( (float*)&(ans_cur->y[0]),c0); + _mm256_store_ps( (float*)&(ans_cur->y[1]),c1); + _mm256_store_ps( (float*)&(ans_cur->y[2]),c2); + _mm256_store_ps( (float*)&(ans_cur->y[3]),c3); } @@ -144,24 +145,24 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) { __mk_param_std(TILE, LEFT_RIGHT_ANS, float); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); - register __m512 c0 = _mm512_setzero_ps(); - register __m512 c1 = _mm512_setzero_ps(); - register __m512 c2 = _mm512_setzero_ps(); - register __m512 c3 = _mm512_setzero_ps(); + register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]); + register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]); + register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]); + register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]); for (size_t z = 0; z < common_z; z+=4) { const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z; - register __m512 b0 = _mm512_load_ps((float *)&b_base[0].z[0]); - register __m512 b1 = _mm512_load_ps((float *)&b_base[1].z[0]); - register __m512 b2 = _mm512_load_ps((float *)&b_base[2].z[0]); - register __m512 b3 = _mm512_load_ps((float *)&b_base[3].z[0]); + register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]); + register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]); + register __m512 b2 = _mm512_load_ps((float *)&b_base->z[2]); + register __m512 b3 = _mm512_load_ps((float *)&b_base->z[3]); - register __m512 a0 = _mm512_load_ps((float *)&a_base[0].z[0]); - register __m512 a1 = _mm512_load_ps((float *)&a_base[1].z[0]); - register __m512 a2 = _mm512_load_ps((float *)&a_base[2].z[0]); - register __m512 a3 = _mm512_load_ps((float *)&a_base[3].z[0]); + register __m512 a0 = _mm512_load_ps((float *)&a_base->z[0]); + register __m512 a1 = _mm512_load_ps((float *)&a_base->z[1]); + register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]); + register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]); SN_MK_FMA_F32_512_CYCLE(0); SN_MK_FMA_F32_512_CYCLE(1); @@ -169,10 +170,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) SN_MK_FMA_F32_512_CYCLE(3); } - SN_FMA_F32_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); - SN_FMA_F32_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); - SN_FMA_F32_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); - SN_FMA_F32_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); + _mm512_store_ps((float *)&ans_cur->z[0], c0); + _mm512_store_ps((float *)&ans_cur->z[1], c1); + _mm512_store_ps((float *)&ans_cur->z[2], c2); + _mm512_store_ps((float *)&ans_cur->z[3], c3); } @@ -350,10 +351,10 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) __mk_param_std(TILE, LEFT_RIGHT_ANS, double); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); - register __m512d c0 = _mm512_setzero_pd(); - register __m512d c1 = _mm512_setzero_pd(); - register __m512d c2 = _mm512_setzero_pd(); - register __m512d c3 = _mm512_setzero_pd(); + register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]); + register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]); + register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]); + register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]); for (size_t z = 0; z < common_z; z+=2) { SN_MAT_KERNEL_LOCAL const *right_cur = right + z; @@ -364,10 +365,10 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]); register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]); - register __m512d b0 = _mm512_load_pd((double *)&left_cur->z[0]); - register __m512d b1 = _mm512_load_pd((double *)&left_cur->z[1]); - register __m512d b2 = _mm512_load_pd((double *)&left_cur->z[2]); - register __m512d b3 = _mm512_load_pd((double *)&left_cur->z[3]); + register __m512d b0 = _mm512_load_pd((double *)&right_cur->z[0]); + register __m512d b1 = _mm512_load_pd((double *)&right_cur->z[1]); + register __m512d b2 = _mm512_load_pd((double *)&right_cur->z[2]); + register __m512d b3 = _mm512_load_pd((double *)&right_cur->z[3]); SN_MK_FMA_F64_AVX2_CYCLE(0); SN_MK_FMA_F64_AVX2_CYCLE(1); @@ -375,6 +376,8 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) SN_MK_FMA_F64_AVX2_CYCLE(3); } _mm512_store_pd((double *)&ans_cur->z[0], c0); - _mm512_store_pd((double *)&ans_cur->z[1], c2); + _mm512_store_pd((double *)&ans_cur->z[1], c1); + _mm512_store_pd((double *)&ans_cur->z[2], c2); + _mm512_store_pd((double *)&ans_cur->z[3], c3); } #endif diff --git a/StgSharp.Native/sn-template/sn_template.fma.x86.h b/StgSharp.Native/sn-template/sn_template.fma.x86.h index 2002847e..47f4f699 100644 --- a/StgSharp.Native/sn-template/sn_template.fma.x86.h +++ b/StgSharp.Native/sn-template/sn_template.fma.x86.h @@ -2,55 +2,36 @@ #if SN_IS_ARCH(SN_ARCH_X86_64) -#define SN_MK_FMA_F32_512_CYCLE(i) \ - do { \ - c0 = _mm512_fmadd_ps(c0, a0, _mm512_permute_ps(b0, _FMA_SELECT(i))); \ - c1 = _mm512_fmadd_ps(c1, a1, _mm512_permute_ps(b1, _FMA_SELECT(i))); \ - c2 = _mm512_fmadd_ps(c2, a2, _mm512_permute_ps(b2, _FMA_SELECT(i))); \ - c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \ +#define SN_MK_FMA_F32_512_CYCLE(i) \ + do { \ + c0 = _mm512_fmadd_ps(a0, _mm512_permute_ps(b0, _FMA_SELECT(i)), c0); \ + c1 = _mm512_fmadd_ps(a1, _mm512_permute_ps(b1, _FMA_SELECT(i)), c1); \ + c2 = _mm512_fmadd_ps(a2, _mm512_permute_ps(b2, _FMA_SELECT(i)), c2); \ + c3 = _mm512_fmadd_ps(a3, _mm512_permute_ps(b3, _FMA_SELECT(i)), c3); \ } while (0) -#define SN_MK_FMA_F32_AVX2_CYCLE(i) \ - do { \ - c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \ - c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \ - c2 = _mm256_fmadd_ps(c2, a2, _mm256_permute_ps(b2, _FMA_SELECT(i))); \ - c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \ +#define SN_MK_FMA_F32_AVX2_FMA_CYCLE(i) \ + do { \ + c0 = _mm256_fmadd_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i)), c0); \ + c1 = _mm256_fmadd_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i)), c1); \ + c2 = _mm256_fmadd_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i)), c2); \ + c3 = _mm256_fmadd_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i)), c3); \ } while (0) -#define SN_MK_FMA_F64_AVX2_CYCLE(i) \ - do { \ - c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \ - c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \ - c2 = _mm512_fmadd_ps(c2, a2, _mm512_permutex_pd(b2, _FMA_SELECT(i))); \ - c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \ +#define SN_MK_FMA_F32_AVX2_CYCLE(i) \ + do { \ + c0 = _mm256_add_ps(_mm256_mul_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i))), c0); \ + c1 = _mm256_add_ps(_mm256_mul_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i))), c1); \ + c2 = _mm256_add_ps(_mm256_mul_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i))), c2); \ + c3 = _mm256_add_ps(_mm256_mul_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i))), c3); \ } while (0) -#define SN_FMA_512_PACK_COLUMN(col, target) \ - do { \ - register __m256 low = _mm512_castps512_ps256(col); \ - register __m256 high = _mm512_extractf32x8_ps(col, 1); \ - register __m256 half = _mm256_add_ps(high, low); \ - register __m128 high128 = _mm256_extractf32x4_ps(half, 1); \ - register __m128 low128 = _mm256_castps256_ps128(half); \ - register __m128 half128 = _mm_add_ps(high128, low128); \ - _mm_store_ps(target, half128); \ - } while (0) - -#define SN_FMA_F64_AVX2_PACK_COLUMN(col, target) \ - do { \ - register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \ - register __m128 low128 = _mm256_castps256_ps128(col); \ - register __m128 half128 = _mm_add_ps(high128, low128); \ - _mm_store_ps(target, half128); \ - } while (0) - -#define SN_FMA_AVX2_PACK_COLUMN(col, target) \ - do { \ - register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \ - register __m128 low128 = _mm256_castps256_ps128(col); \ - register __m128 half128 = _mm_add_ps(high128, low128); \ - _mm_store_ps(target, half128); \ +#define SN_MK_FMA_F64_AVX2_CYCLE(i) \ + do { \ + c0 = _mm512_fmadd_ps(a0, _mm512_permutex_pd(b0, _FMA_SELECT(i)), c0); \ + c1 = _mm512_fmadd_ps(a1, _mm512_permutex_pd(b1, _FMA_SELECT(i)), c1); \ + c2 = _mm512_fmadd_ps(a2, _mm512_permutex_pd(b2, _FMA_SELECT(i)), c2); \ + c3 = _mm512_fmadd_ps(a3, _mm512_permutex_pd(b3, _FMA_SELECT(i)), c3); \ } while (0) // double