youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

left and right use diff pack kernel

Browse Source
This commit is contained in:
Youmu Liu
2026-06-12 18:25:04 +08:00
parent 06a119cc6f
commit 2741b34583
6 changed files with 390 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/mk.fma.x86.c
+18 -105
View File
@@ -1,5 +1,6 @@
#include "sn_target.h"
/* clang-format off */
#include <immintrin.h>
#if SN_IS_ARCH(SN_ARCH_X86_64)
// #include "sn_intrinsic.h"
@@ -11,89 +12,6 @@
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// float
/*
* pack left matrix data to L4 buffer
* scalar pack carries column length and offset data:
* [column length, buffer requires, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i=0;
for(; i <= count_1; i+=2) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if(count_1 == i) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i=0;
for(; i <= count_1; i+=2) {
register __m512 ker_0_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
register __m512 ker_0_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
register __m512 ker_1_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
register __m512 ker_1_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
register __m512 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m512 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m512 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m512 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if(count_1 == i) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
@@ -178,12 +96,11 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_FMA_F32_AVX2_CYCLE(1);
SN_MK_FMA_F32_AVX2_CYCLE(2);
SN_MK_FMA_F32_AVX2_CYCLE(3);
}
_mm256_store_ps( (float*)&(ans_cur->y[0]),c0);
_mm256_store_ps( (float*)&(ans_cur->y[1]),c1);
_mm256_store_ps( (float*)&(ans_cur->y[2]),c2);
_mm256_store_ps( (float*)&(ans_cur->y[3]),c3);
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
@@ -195,7 +112,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z+=2) {
for (size_t z = 0; z < count_0; z += 2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -213,15 +130,13 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_FMA_F32_AVX2_FMA_CYCLE(1);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(2);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(3);
}
_mm256_store_ps( (float*)&(ans_cur->y[0]),c0);
_mm256_store_ps( (float*)&(ans_cur->y[1]),c1);
_mm256_store_ps( (float*)&(ans_cur->y[2]),c2);
_mm256_store_ps( (float*)&(ans_cur->y[3]),c3);
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
@@ -231,7 +146,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < count_0; z+=4) {
for (size_t z = 0; z < count_0; z += 4) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -249,15 +164,13 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_FMA_F32_512_CYCLE(1);
SN_MK_FMA_F32_512_CYCLE(2);
SN_MK_FMA_F32_512_CYCLE(3);
}
_mm512_store_ps((float *)&ans_cur->z[0], c0);
_mm512_store_ps((float *)&ans_cur->z[1], c1);
_mm512_store_ps((float *)&ans_cur->z[2], c2);
_mm512_store_ps((float *)&ans_cur->z[3], c3);
_mm512_store_ps((float *)&ans_cur->z[0], c0);
_mm512_store_ps((float *)&ans_cur->z[1], c1);
_mm512_store_ps((float *)&ans_cur->z[2], c2);
_mm512_store_ps((float *)&ans_cur->z[3], c3);
}
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
@@ -436,7 +349,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < count_0; z+=2) {
for (size_t z = 0; z < count_0; z += 2) {
SN_MAT_KERNEL_LOCAL const *right_base = right + z;
SN_MAT_KERNEL_LOCAL const *left_base = left + z;
@@ -449,7 +362,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1);
SN_MK_FMA_F64_AVX2_CYCLE(2);