diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index 36cb31c8..c5a0459f 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -107,34 +107,36 @@ SN_MK_PROC_DECL_STD(float, avx, , fma) SN_MK_PROC_DECL_STD(float, avx, _fma, fma) { __mk_param_std(TILE, LEFT_RIGHT_ANS, float); + SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); + register __m256 c0 = _mm256_setzero_ps(); + register __m256 c1 = _mm256_setzero_ps(); + register __m256 c2 = _mm256_setzero_ps(); + register __m256 c3 = _mm256_setzero_ps(); - SN_MAT_KERNEL_LOCAL *restrict ans_cur = - ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z+=2) { + const SN_MAT_KERNEL_LOCAL *a_base = left + z; + const SN_MAT_KERNEL_LOCAL *b_base = right + z; - register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]); + register __m256 b0 = _mm256_load_ps((float *)&b_base[0].z[0]); + register __m256 b1 = _mm256_load_ps((float *)&b_base[1].z[0]); + register __m256 b2 = _mm256_load_ps((float *)&b_base[2].z[0]); + register __m256 b3 = _mm256_load_ps((float *)&b_base[3].z[0]); - for (size_t z = 0; z < common_z; z++) { - SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z); - SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y); - register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]); + register __m256 a0 = _mm256_load_ps((float *)&a_base[0].z[0]); + register __m256 a1 = _mm256_load_ps((float *)&a_base[1].z[0]); + register __m256 a2 = _mm256_load_ps((float *)&a_base[2].z[0]); + register __m256 a3 = _mm256_load_ps((float *)&a_base[3].z[0]); + + SN_MK_FMA_F32_AVX2_CYCLE(0); + SN_MK_FMA_F32_AVX2_CYCLE(1); + SN_MK_FMA_F32_AVX2_CYCLE(2); + SN_MK_FMA_F32_AVX2_CYCLE(3); - register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]); - c01 = _mm256_fmadd_ps(a0, b01, c01); - c23 = _mm256_fmadd_ps(a0, b23, c23); - register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]); - c01 = _mm256_fmadd_ps(a1, b01, c01); - c23 = _mm256_fmadd_ps(a1, b23, c23); - register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]); - c01 = _mm256_fmadd_ps(a2, b01, c01); - c23 = _mm256_fmadd_ps(a2, b23, c23); - register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]); - c01 = _mm256_fmadd_ps(a3, b01, c01); - c23 = _mm256_fmadd_ps(a3, b23, c23); } - _mm256_store_ps((float *)&ans_cur->y[0], c01); - _mm256_store_ps((float *)&ans_cur->y[1], c23); + SN_FMA_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); + SN_FMA_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); + SN_FMA_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); + SN_FMA_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); } @@ -167,10 +169,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) SN_MK_FMA_F32_512_CYCLE(3); } - SN_FMA_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); - SN_FMA_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); - SN_FMA_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); - SN_FMA_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); + SN_FMA_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); + SN_FMA_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); + SN_FMA_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); + SN_FMA_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); } diff --git a/StgSharp.Native/sn-template/sn_template.fma.x86.h b/StgSharp.Native/sn-template/sn_template.fma.x86.h index 9267f06a..8dfce50a 100644 --- a/StgSharp.Native/sn-template/sn_template.fma.x86.h +++ b/StgSharp.Native/sn-template/sn_template.fma.x86.h @@ -10,7 +10,15 @@ c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \ } while (0) -#define SN_FMA_PACK_COLUMN(col, target) \ +#define SN_MK_FMA_F32_AVX2_CYCLE(i) \ + do { \ + c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \ + c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \ + c2 = _mm256_fmadd_ps(c2, a2, _mm256_permute_ps(b2, _FMA_SELECT(i))); \ + c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \ + } while (0) + +#define SN_FMA_512_PACK_COLUMN(col, target) \ do { \ register __m256 low = _mm512_castps512_ps256(col); \ register __m256 high = _mm512_extractf32x8_ps(col, 1); \ @@ -21,6 +29,14 @@ _mm_store_ps(target, half128); \ } while (0) +#define SN_FMA_AVX2_PACK_COLUMN(col, target) \ + do { \ + register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \ + register __m128 low128 = _mm256_castps256_ps128(col); \ + register __m128 half128 = _mm_add_ps(high128, low128); \ + _mm_store_ps(target, half128); \ + } while (0) + // double #define SN_MK_FMA_F64_SSE_PHASE(i) \ do { \