youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better kernel fma

Browse Source
This commit is contained in:
Youmu Liu
2026-01-07 18:06:36 +08:00
parent 3d488abd2e
commit 88af8d8957
11 changed files with 347 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/m.fma.x86.c
+153 -54
View File
@@ -7,8 +7,8 @@
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
// float
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
{
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
@@ -57,56 +57,18 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
ans->f32_x[3] = c3;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a03, b23_perm, c23);
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a03, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
@@ -125,49 +87,186 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a03, b23_perm), c23);
c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23);
//group 3
a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
a03 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_add_ps(_mm256_mul_ps(a03, b01_perm), c01);
c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
b01 = _mm256_permutevar_ps(b01, b01_shift);
b23 = _mm256_permutevar_ps(b23, b23_shift);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
//group 0
register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0));
register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0));
c01 = _mm256_fmadd_ps(a01, b01_perm, c01);
c23 = _mm256_fmadd_ps(a23, b23_perm, c23);
//group 2
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2));
c01 = _mm256_fmadd_ps(a23, b01_perm, c01);
c23 = _mm256_fmadd_ps(a01, b23_perm, c23);
//group 1
register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21);
register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30);
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1));
c01 = _mm256_fmadd_ps(a12, b01_perm, c01);
c23 = _mm256_fmadd_ps(a30, b23_perm, c23);
//group 3
b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3));
b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3));
c01 = _mm256_fmadd_ps(a30, b01_perm, c01);
c23 = _mm256_fmadd_ps(a12, b23_perm, c23);
ans->f32_y[0] = c01;
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, 12, 13, 14 };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
register __m512 c = _mm512_load_ps((float *)ans);
b = _mm512_permutexvar_ps(b_shift, b);
register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0));
register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(1));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(2));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1));
b_perm = _mm512_permute_ps(b, _FMA_SELECT(3));
c = _mm512_fmadd_ps(a, b_perm, c);
c = _mm512_fmadd_ps(a_perm, b_perm, c);
_mm512_store_ps((float *)ans, c);
}
// double
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
{
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]);
__m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]);
__m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]);
__m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]);
__m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]);
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {