youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

f64 fma turns to be kernel mode

Browse Source
This commit is contained in:
Youmu Liu
2026-01-08 18:16:14 +08:00
parent 88af8d8957
commit 4ff1d856a7
3 changed files with 87 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrincontext.x86.h
+4 -4
View File
@@ -154,10 +154,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
#pragma endregion
StgSharp.Native/sn-src/load_intrinsic.x86.c
+4 -4
View File
@@ -63,7 +63,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, sse, , fma);
context->mat[F64].panel_fma = KER_PROC(double, sse, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
@@ -99,8 +99,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
context->mat[F64].panel_fma = is_fma ? PNL_PROC(double, avx, , fma) :
PNL_PROC(double, avx, _fma, fma);
context->mat[F64].panel_fma = is_fma ? KER_PROC(double, avx, , fma) :
KER_PROC(double, avx, _fma, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
@@ -131,7 +131,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = BUILD_PANEL(512, double);
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, 512, , fma);
context->mat[F64].panel_fma = KER_PROC(double, 512, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = STORE_PANEL(512, double);
StgSharp.Native/sn-src/m.fma.x86.c
+79 -40
View File
@@ -59,8 +59,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
@@ -105,8 +105,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 };
static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 };
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]);
register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]);
@@ -151,7 +151,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
static const __m512i b_shift = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, 12, 13, 14 };
static const __m512i b_shift = { .m512i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15,
12, 13, 14 } };
register __m512 a = _mm512_load_ps((float *)left);
register __m512 b = _mm512_load_ps((float *)right);
@@ -267,55 +268,93 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
ans->f64_y[0] = ccol;
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
{
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
register __m512d a01 = _mm512_load_pd((double *)&left->f64_z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->f64_z[1]);
register __m512d b01 = _mm512_load_pd((double *)&right->f64_z[0]);
register __m512d b23 = _mm512_load_pd((double *)&right->f64_z[1]);
b01 = _mm512_permutexvar_pd(b01_shift, b01);
b23 = _mm512_permutexvar_pd(b23_shift, b23);
register __m512d c01 = _mm512_load_pd((float *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((float *)&ans->f64_z[1]);
//group 0
register __m512d b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(0));
register __m512d b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(0));
c01 = _mm512_fmadd_pd(a01, b01_perm, c01);
c23 = _mm512_fmadd_pd(a23, b23_perm, c23);
//group 2
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(2));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(2));
c01 = _mm512_fmadd_pd(a23, b01_perm, c01);
c23 = _mm512_fmadd_pd(a01, b23_perm, c23);
//group 1
register __m512d a12 = _mm512_shuffle_f64x2(a01, a23, _MM_SHUFFLE(1, 0, 3, 2));
register __m512d a30 = _mm512_shuffle_f64x2(a23, a01, _MM_SHUFFLE(1, 0, 3, 2));
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(1));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(1));
c01 = _mm512_fmadd_pd(a12, b01_perm, c01);
c23 = _mm512_fmadd_pd(a30, b23_perm, c23);
//group 3
b01_perm = _mm512_shuffle_pd(b01, b01, _FMA_SELECT(3));
b23_perm = _mm512_shuffle_pd(b23, b23, _FMA_SELECT(3));
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
ans->f64_z[0] = c01;
ans->f64_z[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
register __m128d c_lo = _mm_castps_pd(PANEL_COL_VEC((*ans), col, 0));
register __m128d c_hi = _mm_castps_pd(PANEL_COL_VEC((*ans), col, 1));
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
register __m128d c_hi = _mm_load_pd(ans->f64_x + (col * 2 + 1));
/* right matrix column split into two chunks */
register __m128d b_lo = _mm_castps_pd(PANEL_COL_VEC((*right), col, 0));
register __m128d b_hi = _mm_castps_pd(PANEL_COL_VEC((*right), col, 1));
register __m128d b_lo = _mm_load_pd(right + (2 * col + 0));
register __m128d b_hi = _mm_load_pd(right + (2 * col + 1));
/* k = 0 */
{
register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 0, 0));
register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 0, 1));
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
}
register __m128d a_lo = _mm_load_pd(left + (0 * 2, 0));
register __m128d a_hi = _mm_load_pd(left + (0 * 2, 1));
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 1 */
{
register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 1, 0));
register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 1, 1));
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
}
a_lo = _mm_load_pd(left + (1 * 2, 0));
a_hi = _mm_load_pd(left + (1 * 2, 1));
b = _mm_shuffle_pd(b_lo, b_lo, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 2 */
{
register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 2, 0));
register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 2, 1));
register __m128d b = _mm_shuffle_pd(b_hi, b_hi, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
}
a_lo = _mm_load_pd(left + (1 * 2, 0));
a_hi = _mm_load_pd(left + (1 * 2, 1));
b = _mm_shuffle_pd(b_hi, b_hi, 0x0);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
/* k = 3 */
{
register __m128d a_lo = _mm_castps_pd(PANEL_COL_VEC((*left), 3, 0));
register __m128d a_hi = _mm_castps_pd(PANEL_COL_VEC((*left), 3, 1));
register __m128d b = _mm_shuffle_pd(b_hi, b_hi, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
}
a_lo = _mm_load_pd(left + (1 * 2, 0));
a_hi = _mm_load_pd(left + (1 * 2, 1));
b = _mm_shuffle_pd(b_hi, b_hi, 0x3);
c_lo = _mm_add_pd(c_lo, _mm_mul_pd(a_lo, b));
c_hi = _mm_add_pd(c_hi, _mm_mul_pd(a_hi, b));
PANEL_COL_VEC((*ans), col, 0) = _mm_castpd_ps(c_lo);
PANEL_COL_VEC((*ans), col, 1) = _mm_castpd_ps(c_hi);
ans->f64_x[col * 2 + 0] = c_lo;
ans->f64_x[col * 2 + 1] = c_hi;
}
}