youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tiled fma is ok, mat mul is now available again

Browse Source
This commit is contained in:
Youmu Liu
2026-01-12 15:09:39 +08:00
parent b8bf1e9f08
commit ad3b2212fa
6 changed files with 172 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/m.fma_tile.x86.c
+31 -37
View File
@@ -175,8 +175,8 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
MAT_KERNEL(float)
const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
register __m256i b01_shift = _mm256_load_si256(&b01_shift_source);
register __m256i b23_shift = _mm256_load_si256(&b23_shift_source);
@@ -195,9 +195,6 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
@@ -270,35 +267,35 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
}
// double
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b));
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
@@ -519,9 +516,6 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
b01 = _mm512_permutexvar_pd(b01_shift, b01);
b23 = _mm512_permutexvar_pd(b23_shift, b23);
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
//group 0
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));