youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

other gemm kernel use half-vector-machine style

Browse Source
This commit is contained in:
Youmu Liu
2026-06-08 16:18:03 +08:00
parent 1c515ea2ae
commit 9d5c0fd8b4
4 changed files with 129 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/StgSharpNative.h
+4 -8
View File
@@ -1,10 +1,10 @@
#ifdef _MSC_VER
#pragma once
#endif
#ifndef SNative #ifndef SNative
#define SNative #define SNative
#ifdef __cplusplus
#error "This header is not meant to be included in C++ code. StgSharp.Native is a C library and should be used with C linkage."
#endif
#include "sn_internal.h" #include "sn_internal.h"
#include "sn_target.h" #include "sn_target.h"
@@ -26,10 +26,6 @@
#ifndef SN_ #ifndef SN_
#endif #endif
#define ALIGN(simdVec) _mm_loadu_ps(&simdVec)
#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec)
#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec)
typedef struct Image { typedef struct Image {
int width; int width;
int height; int height;
StgSharp.Native/sn-head/sn_intrinsic.h
+23 -2
View File
@@ -4,6 +4,7 @@
#define SSC_INTRIN_HEAD #define SSC_INTRIN_HEAD
#include "StgSharpNative.h" #include "StgSharpNative.h"
#include "sn_target.h"
#include <xmmintrin.h> #include <xmmintrin.h>
#include <immintrin.h> #include <immintrin.h>
#include <smmintrin.h> #include <smmintrin.h>
@@ -26,9 +27,31 @@
#define M256_PERMUTE(s3, s2, s1, s0) \ #define M256_PERMUTE(s3, s2, s1, s0) \
((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0))) ((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)))
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index) #define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
#if SN_IS_ARCH(x86_64)
#define vec_128 __m128
#define vec_256 __m256
#define vec_512 __m512
#define zero_vec _mm_setzero_ps() #define zero_vec _mm_setzero_ps()
#define zero_vec_256 _mm256_setzero_ps() #define zero_vec_256 _mm256_setzero_ps()
#define ALIGN(simdVec) _mm_loadu_ps(&simdVec)
#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec)
#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec)
#else
#define zero_vec
#define zero_vec_256
#define ALIGN(simdVec)
#define ALIGN256(simdVec)
#define ALIGN512(simdVec)
#endif
#define SPAN(T) \ #define SPAN(T) \
struct SPAN_T { \ struct SPAN_T { \
T *pointer; \ T *pointer; \
@@ -126,6 +149,4 @@ typedef struct sn_intrinsic {
typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask; typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask;
#include "sn_target.h"
#endif #endif
StgSharp.Native/sn-src/mk.fma.x86.c
+78 -75
View File
@@ -71,61 +71,27 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
} }
SN_MK_PROC_DECL_STD(float, avx, , fma) SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->y[1]);
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
register __m256 b01 = _mm256_load_ps((float *)&right_cur->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right_cur->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left_cur->x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left_cur->x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left_cur->x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left_cur->x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
}
_mm256_store_ps((float *)&ans_cur->y[0], c01);
_mm256_store_ps((float *)&ans_cur->y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{ {
__mk_param_std(TILE, LEFT_RIGHT_ANS, float); __mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m256 c0 = _mm256_setzero_ps(); register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_setzero_ps(); register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_setzero_ps(); register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_setzero_ps(); register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < common_z; z+=2) { for (size_t z = 0; z < common_z; z+=2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m256 b0 = _mm256_load_ps((float *)&b_base[0].z[0]); register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base[1].z[0]); register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base[2].z[0]); register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base[3].z[0]); register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base[0].z[0]); register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base[1].z[0]); register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base[2].z[0]); register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base[3].z[0]); register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
SN_MK_FMA_F32_AVX2_CYCLE(0); SN_MK_FMA_F32_AVX2_CYCLE(0);
SN_MK_FMA_F32_AVX2_CYCLE(1); SN_MK_FMA_F32_AVX2_CYCLE(1);
@@ -133,10 +99,45 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_FMA_F32_AVX2_CYCLE(3); SN_MK_FMA_F32_AVX2_CYCLE(3);
} }
SN_FMA_F32_AVX2_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); _mm256_store_ps( (float*)&(ans_cur->y[0]),c0);
SN_FMA_F32_AVX2_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); _mm256_store_ps( (float*)&(ans_cur->y[1]),c1);
SN_FMA_F32_AVX2_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); _mm256_store_ps( (float*)&(ans_cur->y[2]),c2);
SN_FMA_F32_AVX2_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); _mm256_store_ps( (float*)&(ans_cur->y[3]),c3);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < common_z; z+=2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(0);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(1);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(2);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(3);
}
_mm256_store_ps( (float*)&(ans_cur->y[0]),c0);
_mm256_store_ps( (float*)&(ans_cur->y[1]),c1);
_mm256_store_ps( (float*)&(ans_cur->y[2]),c2);
_mm256_store_ps( (float*)&(ans_cur->y[3]),c3);
} }
@@ -144,24 +145,24 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
{ {
__mk_param_std(TILE, LEFT_RIGHT_ANS, float); __mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c0 = _mm512_setzero_ps(); register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c1 = _mm512_setzero_ps(); register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c2 = _mm512_setzero_ps(); register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_setzero_ps(); register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < common_z; z+=4) { for (size_t z = 0; z < common_z; z+=4) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m512 b0 = _mm512_load_ps((float *)&b_base[0].z[0]); register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]);
register __m512 b1 = _mm512_load_ps((float *)&b_base[1].z[0]); register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]);
register __m512 b2 = _mm512_load_ps((float *)&b_base[2].z[0]); register __m512 b2 = _mm512_load_ps((float *)&b_base->z[2]);
register __m512 b3 = _mm512_load_ps((float *)&b_base[3].z[0]); register __m512 b3 = _mm512_load_ps((float *)&b_base->z[3]);
register __m512 a0 = _mm512_load_ps((float *)&a_base[0].z[0]); register __m512 a0 = _mm512_load_ps((float *)&a_base->z[0]);
register __m512 a1 = _mm512_load_ps((float *)&a_base[1].z[0]); register __m512 a1 = _mm512_load_ps((float *)&a_base->z[1]);
register __m512 a2 = _mm512_load_ps((float *)&a_base[2].z[0]); register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]);
register __m512 a3 = _mm512_load_ps((float *)&a_base[3].z[0]); register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]);
SN_MK_FMA_F32_512_CYCLE(0); SN_MK_FMA_F32_512_CYCLE(0);
SN_MK_FMA_F32_512_CYCLE(1); SN_MK_FMA_F32_512_CYCLE(1);
@@ -169,10 +170,10 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_FMA_F32_512_CYCLE(3); SN_MK_FMA_F32_512_CYCLE(3);
} }
SN_FMA_F32_512_PACK_COLUMN(c0, (float*)&(ans_cur->x[0])); _mm512_store_ps((float *)&ans_cur->z[0], c0);
SN_FMA_F32_512_PACK_COLUMN(c1, (float*)&(ans_cur->x[1])); _mm512_store_ps((float *)&ans_cur->z[1], c1);
SN_FMA_F32_512_PACK_COLUMN(c2, (float*)&(ans_cur->x[2])); _mm512_store_ps((float *)&ans_cur->z[2], c2);
SN_FMA_F32_512_PACK_COLUMN(c3, (float*)&(ans_cur->x[3])); _mm512_store_ps((float *)&ans_cur->z[3], c3);
} }
@@ -350,10 +351,10 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
__mk_param_std(TILE, LEFT_RIGHT_ANS, double); __mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c0 = _mm512_setzero_pd(); register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c1 = _mm512_setzero_pd(); register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
register __m512d c2 = _mm512_setzero_pd(); register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_setzero_pd(); register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < common_z; z+=2) { for (size_t z = 0; z < common_z; z+=2) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + z; SN_MAT_KERNEL_LOCAL const *right_cur = right + z;
@@ -364,10 +365,10 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]); register __m512d a2 = _mm512_load_pd((double *)&left_cur->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]); register __m512d a3 = _mm512_load_pd((double *)&left_cur->z[3]);
register __m512d b0 = _mm512_load_pd((double *)&left_cur->z[0]); register __m512d b0 = _mm512_load_pd((double *)&right_cur->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&left_cur->z[1]); register __m512d b1 = _mm512_load_pd((double *)&right_cur->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&left_cur->z[2]); register __m512d b2 = _mm512_load_pd((double *)&right_cur->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&left_cur->z[3]); register __m512d b3 = _mm512_load_pd((double *)&right_cur->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0); SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1); SN_MK_FMA_F64_AVX2_CYCLE(1);
@@ -375,6 +376,8 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
SN_MK_FMA_F64_AVX2_CYCLE(3); SN_MK_FMA_F64_AVX2_CYCLE(3);
} }
_mm512_store_pd((double *)&ans_cur->z[0], c0); _mm512_store_pd((double *)&ans_cur->z[0], c0);
_mm512_store_pd((double *)&ans_cur->z[1], c2); _mm512_store_pd((double *)&ans_cur->z[1], c1);
_mm512_store_pd((double *)&ans_cur->z[2], c2);
_mm512_store_pd((double *)&ans_cur->z[3], c3);
} }
#endif #endif
StgSharp.Native/sn-template/sn_template.fma.x86.h
+24 -43
View File
@@ -2,55 +2,36 @@
#if SN_IS_ARCH(SN_ARCH_X86_64) #if SN_IS_ARCH(SN_ARCH_X86_64)
#define SN_MK_FMA_F32_512_CYCLE(i) \ #define SN_MK_FMA_F32_512_CYCLE(i) \
do { \ do { \
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permute_ps(b0, _FMA_SELECT(i))); \ c0 = _mm512_fmadd_ps(a0, _mm512_permute_ps(b0, _FMA_SELECT(i)), c0); \
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permute_ps(b1, _FMA_SELECT(i))); \ c1 = _mm512_fmadd_ps(a1, _mm512_permute_ps(b1, _FMA_SELECT(i)), c1); \
c2 = _mm512_fmadd_ps(c2, a2, _mm512_permute_ps(b2, _FMA_SELECT(i))); \ c2 = _mm512_fmadd_ps(a2, _mm512_permute_ps(b2, _FMA_SELECT(i)), c2); \
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \ c3 = _mm512_fmadd_ps(a3, _mm512_permute_ps(b3, _FMA_SELECT(i)), c3); \
} while (0) } while (0)
#define SN_MK_FMA_F32_AVX2_CYCLE(i) \ #define SN_MK_FMA_F32_AVX2_FMA_CYCLE(i) \
do { \ do { \
c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \ c0 = _mm256_fmadd_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i)), c0); \
c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \ c1 = _mm256_fmadd_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i)), c1); \
c2 = _mm256_fmadd_ps(c2, a2, _mm256_permute_ps(b2, _FMA_SELECT(i))); \ c2 = _mm256_fmadd_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i)), c2); \
c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \ c3 = _mm256_fmadd_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i)), c3); \
} while (0) } while (0)
#define SN_MK_FMA_F64_AVX2_CYCLE(i) \ #define SN_MK_FMA_F32_AVX2_CYCLE(i) \
do { \ do { \
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \ c0 = _mm256_add_ps(_mm256_mul_ps(a0, _mm256_permute_ps(b0, _FMA_SELECT(i))), c0); \
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \ c1 = _mm256_add_ps(_mm256_mul_ps(a1, _mm256_permute_ps(b1, _FMA_SELECT(i))), c1); \
c2 = _mm512_fmadd_ps(c2, a2, _mm512_permutex_pd(b2, _FMA_SELECT(i))); \ c2 = _mm256_add_ps(_mm256_mul_ps(a2, _mm256_permute_ps(b2, _FMA_SELECT(i))), c2); \
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \ c3 = _mm256_add_ps(_mm256_mul_ps(a3, _mm256_permute_ps(b3, _FMA_SELECT(i))), c3); \
} while (0) } while (0)
#define SN_FMA_512_PACK_COLUMN(col, target) \ #define SN_MK_FMA_F64_AVX2_CYCLE(i) \
do { \ do { \
register __m256 low = _mm512_castps512_ps256(col); \ c0 = _mm512_fmadd_ps(a0, _mm512_permutex_pd(b0, _FMA_SELECT(i)), c0); \
register __m256 high = _mm512_extractf32x8_ps(col, 1); \ c1 = _mm512_fmadd_ps(a1, _mm512_permutex_pd(b1, _FMA_SELECT(i)), c1); \
register __m256 half = _mm256_add_ps(high, low); \ c2 = _mm512_fmadd_ps(a2, _mm512_permutex_pd(b2, _FMA_SELECT(i)), c2); \
register __m128 high128 = _mm256_extractf32x4_ps(half, 1); \ c3 = _mm512_fmadd_ps(a3, _mm512_permutex_pd(b3, _FMA_SELECT(i)), c3); \
register __m128 low128 = _mm256_castps256_ps128(half); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_F64_AVX2_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
register __m128 low128 = _mm256_castps256_ps128(col); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_AVX2_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
register __m128 low128 = _mm256_castps256_ps128(col); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0) } while (0)
// double // double