youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite of f32 fma, and other optmization of simd kernel ops

Browse Source
This commit is contained in:
Youmu Liu
2025-11-10 20:40:19 +08:00
parent 1dee467220
commit 678a846e54
18 changed files with 302 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/StgSharp.Native.vcxproj
+3 -1
View File
@@ -109,7 +109,9 @@
<ClCompile Include="sn-src\str_factorial.c" />
<ClCompile Include="sn-src\k_fma.c" />
<ClCompile Include="sn-src\str_hash.c" />
<ClCompile Include="sn-src\load_intrinsic.c" />
<ClCompile Include="sn-src\load_intrinsic.c">
<FileType>CppCode</FileType>
</ClCompile>
<ClCompile Include="sn-src\load_glfw.c" />
<ClCompile Include="sn-src\stb_image.c" />
<ClCompile Include="sn-src\sn_native.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
+3 -3
View File
@@ -24,9 +24,6 @@
<ClCompile Include="sn-src\str_hash.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_intrinsic.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\stb_image.c">
<Filter>源文件</Filter>
</ClCompile>
@@ -51,6 +48,9 @@
<ClCompile Include="sn-src\k_fma.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_intrinsic.c">
<Filter>源文件</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="sn-head\gl.h">
StgSharp.Native/sn-head/sn_intrinsic.h
+29 -24
View File
@@ -31,12 +31,7 @@ typedef union uint64_u {
uint32_t u32[2];
} uint64_u;
typedef union mat_kernel {
__m128 xmm[4];
__m256 ymm[2];
__m512 zmm[1];
float m[4][4];
} mat_kernel;
typedef struct mat_kernel mat_kernel;
typedef union scalar_pack {
uint64_t data[8];
@@ -44,13 +39,10 @@ typedef union scalar_pack {
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef void(SN_DECL *MATRIXTRANSPOSEPROC)(void *source, void *target);
typedef void(SN_DECL *MATRIXDETPROC)(void *mat, void *transpose);
typedef void(SN_DECL *MATARITHMETIC)(void *left, void *right, void *ans);
typedef void(SN_DECL *SCALERMATARITHMETIC)(void *left, void *right, void *ans);
typedef void(SN_DECL *MATFMAPROC)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *SCALARMATARITHMETIC)(mat_kernel *matrix, float scalar, mat_kernel *ans);
typedef void(SN_DECL *MATTRANSPOSEPROC)(mat_kernel *source, mat_kernel *target);
typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
#pragma region matix function
@@ -75,12 +67,18 @@ INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
#pragma endregion
#pragma region fma
@@ -88,6 +86,12 @@ INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
#pragma endregion
@@ -119,11 +123,12 @@ typedef struct sn_intrinsic {
int (*city_hash_simplify)(byte *str, int length);
FACTORIALROC factorial_simd;
int (*index_pair)(short const *str, uint32_t target, int length);
MATARITHMETIC f32_add;
void (*f32_fma)(void *left, void *right, void *ans);
SCALARMATARITHMETIC f32_scalar_mul;
MATARITHMETIC f32_sub;
MATTRANSPOSEPROC f32_transpose;
MAT_PROC_LRA f32_add;
MAT_PROC_AS f32_fill;
MAT_PROC_LRA f32_fma;
MAT_PROC_LRA f32_scalar_mul;
MAT_PROC_LRA f32_sub;
MAT_PROC_LA f32_transpose;
VECTORNORMALIZEPROC f32_normalize_3;
} sn_intrinsic;
StgSharp.Native/sn-src/k_basicop.c
+31 -10
View File
@@ -1,5 +1,27 @@
#include "sn_intrinsic.h"
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
{
register __m128 vec = _mm_set_ps1(((float *)(scaler->data))[0]);
dest->xmm[0] = vec;
dest->xmm[1] = vec;
dest->xmm[2] = vec;
dest->xmm[3] = vec;
}
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scaler->data)));
dest->ymm[0] = vec;
dest->ymm[1] = vec;
}
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scaler->data))[0]);
dest->zmm[0] = vec;
}
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans)
{
@@ -106,11 +128,10 @@ INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
target->zmm[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
{
register __m128 temp = _mm_load_ss(&scalar);
register __m128 scalar_vec = _mm_shuffle_ps(temp, temp, _MM_SHUFFLE(0, 0, 0, 0));
register __m128 scalar_vec = ((__m128 *)(scalar->data))[0];
register __m128 c0 = _mm_mul_ps(matrix->xmm[0], scalar_vec);
register __m128 c1 = _mm_mul_ps(matrix->xmm[1], scalar_vec);
@@ -123,10 +144,10 @@ INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix, float
ans->xmm[3] = c3;
}
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
{
register __m256 scalar_vec = _mm256_broadcast_ss(&scalar);
register __m256 scalar_vec = ((__m256 *)(scalar->data))[0];
register __m256 c0 = _mm256_mul_ps(matrix->ymm[0], scalar_vec);
register __m256 c1 = _mm256_mul_ps(matrix->ymm[1], scalar_vec);
@@ -135,10 +156,10 @@ INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix, float
ans->ymm[1] = c1;
}
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix, float const scalar,
MAT_KERNEL(float) *restrict ans)
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans, scalar_pack const *scalar)
{
__m512 scalar_vec = _mm512_broadcastss_ps(_mm_load_ss(&scalar));
register __m512 scalar_vec = ((__m512 *)(scalar->data))[0];
register __m512 c0 = _mm512_mul_ps(matrix->zmm[0], scalar_vec);
StgSharp.Native/sn-src/k_fma.c
+39 -25
View File
@@ -1,6 +1,7 @@
#include "sn_intrinsic.h"
// ans = left * right + ans
// right is not transposed
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
@@ -10,34 +11,47 @@ INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
register __m128 source2 = _mm_load_ps(&left->xmm[2]);
register __m128 source3 = _mm_load_ps(&left->xmm[3]);
register __m128 ans0 = _mm_load_ps(&ans->xmm[0]);
register __m128 ans1 = _mm_load_ps(&ans->xmm[1]);
register __m128 ans2 = _mm_load_ps(&ans->xmm[2]);
register __m128 ans3 = _mm_load_ps(&ans->xmm[3]);
register __m128 broadcast;
register __m128 ans0 = _mm_setzero_ps();
register __m128 ans1 = _mm_setzero_ps();
register __m128 ans2 = _mm_setzero_ps();
register __m128 ans3 = _mm_setzero_ps();
for (size_t i = 0; i < 3; i++) {
for (size_t i = 0; i < 4; i++) {
broadcast = _mm_load_ps(&right->xmm[i]);
_mm_add_ps(ans0, _mm_mul_ps(source0, broadcast));
_mm_add_ps(ans1, _mm_mul_ps(source1, broadcast));
_mm_add_ps(ans2, _mm_mul_ps(source2, broadcast));
_mm_add_ps(ans3, _mm_mul_ps(source3, broadcast));
for (size_t t = 0; t < 2; t++) {
broadcast = _mm_shuffle_ps(broadcast, broadcast, _MM_SHUFFLE(2, 1, 0, 3));
_mm_add_ps(ans0, _mm_mul_ps(source0, broadcast));
_mm_add_ps(ans1, _mm_mul_ps(source1, broadcast));
_mm_add_ps(ans2, _mm_mul_ps(source2, broadcast));
_mm_add_ps(ans3, _mm_mul_ps(source3, broadcast));
}
ans0 = _mm_add_ps(ans0,
_mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00)));
ans1 = _mm_add_ps(ans1,
_mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55)));
ans2 = _mm_add_ps(ans2,
_mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA)));
ans3 = _mm_add_ps(ans3,
_mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF)));
}
source0 = _mm_load_ps(&ans->xmm[0]);
source1 = _mm_load_ps(&ans->xmm[1]);
source2 = _mm_load_ps(&ans->xmm[2]);
source3 = _mm_load_ps(&ans->xmm[3]);
_mm_store_ps(&ans->xmm[0], _mm_add_ps(source0, ans0));
_mm_store_ps(&ans->xmm[1], _mm_add_ps(source1, ans1));
_mm_store_ps(&ans->xmm[2], _mm_add_ps(source2, ans2));
_mm_store_ps(&ans->xmm[3], _mm_add_ps(source3, ans3));
_mm_store_ps(&ans->xmm[0], ans0);
_mm_store_ps(&ans->xmm[1], ans1);
_mm_store_ps(&ans->xmm[2], ans2);
_mm_store_ps(&ans->xmm[3], ans3);
}
// ans = left * right + ans
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
{
/*
* to be implemented
*/
}
// ans = left * right + ans
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans)
{
/*
* to be implemented
*/
}
StgSharp.Native/sn-src/load_intrinsic.c
+12 -12
View File
@@ -65,11 +65,11 @@ void load_intrinsic_sse(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MATARITHMETIC)f32_add_sse;
context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_sse;
context->f32_fma = (MAT_PROC_LRA)f32_fma_sse;
context->f32_scalar_mul = f32_scalar_mul_sse;
context->f32_sub = (MATARITHMETIC)f32_sub_sse;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_sse;
context->f32_sub = (MAT_PROC_LRA)f32_sub_sse;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_sse;
context->f32_normalize_3 = f32_normalize;
}
@@ -78,11 +78,11 @@ void load_intrinsic_avx(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MATARITHMETIC)f32_add_avx;
context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_avx;
context->f32_fma = (MAT_PROC_LRA)f32_fma_avx;
context->f32_scalar_mul = f32_scalar_mul_avx;
context->f32_sub = (MATARITHMETIC)f32_sub_avx;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_avx;
context->f32_sub = (MAT_PROC_LRA)f32_sub_avx;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_avx;
context->f32_normalize_3 = f32_normalize;
}
@@ -91,10 +91,10 @@ void load_intrinsic_512(sn_intrinsic *context)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->f32_add = (MATARITHMETIC)f32_add_512;
context->f32_fma = (void (*)(void *, void *, void *))f32_fma_sse;
context->f32_add = (MAT_PROC_LRA)f32_add_512;
context->f32_fma = (MAT_PROC_LRA)f32_fma_512;
context->f32_scalar_mul = f32_scalar_mul_512;
context->f32_sub = (MATARITHMETIC)f32_sub_512;
context->f32_transpose = (MATTRANSPOSEPROC)f32_transpose_512;
context->f32_sub = (MAT_PROC_LRA)f32_sub_512;
context->f32_transpose = (MAT_PROC_LA)f32_transpose_512;
context->f32_normalize_3 = f32_normalize;
}