youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gemm/Fma kernel on avx2 use new solution, with less permute and shuff instruction

Browse Source
This commit is contained in:
Youmu Liu
2026-06-04 21:05:38 +08:00
parent 2690cf9341
commit 94b2d759ae
2 changed files with 46 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/mk.fma.x86.c
+29 -27
View File
@@ -107,34 +107,36 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m256 c0 = _mm256_setzero_ps();
register __m256 c1 = _mm256_setzero_ps();
register __m256 c2 = _mm256_setzero_ps();
register __m256 c3 = _mm256_setzero_ps();
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
for (size_t z = 0; z < common_z; z+=2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 b0 = _mm256_load_ps((float *)&b_base[0].z[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base[1].z[0]);
register __m256 b2 = _mm256_load_ps((float *)&b_base[2].z[0]);
register __m256 b3 = _mm256_load_ps((float *)&b_base[3].z[0]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]);
register __m256 a0 = _mm256_load_ps((float *)&a_base[0].z[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base[1].z[0]);
register __m256 a2 = _mm256_load_ps((float *)&a_base[2].z[0]);
register __m256 a3 = _mm256_load_ps((float *)&a_base[3].z[0]);
SN_MK_FMA_F32_AVX2_CYCLE(0);
SN_MK_FMA_F32_AVX2_CYCLE(1);
SN_MK_FMA_F32_AVX2_CYCLE(2);
SN_MK_FMA_F32_AVX2_CYCLE(3);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
}
_mm256_store_ps((float *)&ans_cur->y[0], c01);
_mm256_store_ps((float *)&ans_cur->y[1], c23);
SN_FMA_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
}
@@ -167,10 +169,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_FMA_F32_512_CYCLE(3);
}
SN_FMA_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
SN_FMA_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
}