youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FMA-avx512-f64 kernel use half-vector-machine design

Browse Source
This commit is contained in:
Nitload
2026-06-08 01:02:34 +08:00
parent 4336b0ca5a
commit 1c515ea2ae
5 changed files with 51 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/.clangd
+2
View File
@@ -0,0 +1,2 @@
CompileFlags:
CompilationDatabase: .
StgSharp.Native/sn-head/sn_intrinsic.h
+2 -1
View File
@@ -4,9 +4,10 @@
#define SSC_INTRIN_HEAD
#include "StgSharpNative.h"
#include "sn_intrinsic.matkernel.h"
#include <xmmintrin.h>
#include <immintrin.h>
#include <smmintrin.h>
#include "sn_intrinsic.matkernel.h"
#include <stdint.h>
#ifdef _MSC_VER
StgSharp.Native/sn-head/sn_intrinsic.matkernel.h
+1
View File
@@ -22,6 +22,7 @@
#define SN_DEFINE_MAT_KERNEL_TYPE(name, T) \
typedef union name SN_MAT_KERNEL_MEMBER(T) name
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_float, float);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_double, double);
SN_DEFINE_MAT_KERNEL_TYPE(sn_mat_kernel_uint64_t, uint64_t);
StgSharp.Native/sn-src/mk.fma.x86.c
+30 -33
View File
@@ -133,10 +133,10 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_FMA_F32_AVX2_CYCLE(3);
}
SN_FMA_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
SN_FMA_F32_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_F32_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_F32_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_F32_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
}
@@ -169,10 +169,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_FMA_F32_512_CYCLE(3);
}
SN_FMA_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
SN_FMA_F32_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0]));
SN_FMA_F32_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1]));
SN_FMA_F32_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2]));
SN_FMA_F32_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3]));
}
@@ -348,36 +348,33 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c01 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans_cur->z[1]);
register __m512d c0 = _mm512_setzero_pd();
register __m512d c1 = _mm512_setzero_pd();
register __m512d c2 = _mm512_setzero_pd();
register __m512d c3 = _mm512_setzero_pd();
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
for (size_t z = 0; z < common_z; z+=2) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + z;
SN_MAT_KERNEL_LOCAL const *left_cur = left + z;
register __m512d a01 = _mm512_load_pd((double *)&left_cur->z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left_cur->z[1]);
register __m512d a0 = _mm512_load_pd((double *)&left_cur->z[0]);
register __m512d a1 = _mm512_load_pd((double *)&left_cur->z[1]);
register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]);
register __m512d b0 = _mm512_broadcast_f64x4(right_cur->y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right_cur->y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right_cur->y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right_cur->y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
register __m512d b0 = _mm512_load_pd((double *)&left_cur->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&left_cur->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&left_cur->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&left_cur->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1);
SN_MK_FMA_F64_AVX2_CYCLE(2);
SN_MK_FMA_F64_AVX2_CYCLE(3);
}
_mm512_store_pd((double *)&ans_cur->z[0], c01);
_mm512_store_pd((double *)&ans_cur->z[1], c23);
_mm512_store_pd((double *)&ans_cur->z[0], c0);
_mm512_store_pd((double *)&ans_cur->z[1], c2);
}
#endif
StgSharp.Native/sn-template/sn_template.fma.x86.h
+16
View File
@@ -18,6 +18,14 @@
c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \
} while (0)
#define SN_MK_FMA_F64_AVX2_CYCLE(i) \
do { \
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \
c2 = _mm512_fmadd_ps(c2, a2, _mm512_permutex_pd(b2, _FMA_SELECT(i))); \
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \
} while (0)
#define SN_FMA_512_PACK_COLUMN(col, target) \
do { \
register __m256 low = _mm512_castps512_ps256(col); \
@@ -29,6 +37,14 @@
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_F64_AVX2_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
register __m128 low128 = _mm256_castps256_ps128(col); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_AVX2_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \