youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

decide to use L4 to solve numa and pack problem for large compute.

Browse Source
This commit is contained in:
Youmu Liu
2026-03-02 17:33:03 +08:00
parent 67fbbde449
commit 7e3ed8b0fd
9 changed files with 192 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/load_intrinsic.x86.c
+45 -50
View File
@@ -4,7 +4,8 @@
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
@@ -51,24 +52,22 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, sse, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose);
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
context->mat[F32].ker_fma = KER_PROC(float, sse, , fma);
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, sse, , fma);
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, sse, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
context->mat[F64].ker_fma = KER_PROC(double, sse, , fma);
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, sse, , fma);
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
@@ -84,28 +83,26 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, avx, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose);
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
context->mat[F32].ker_fma = is_fma ? KER_PROC(float, avx, , fma) :
KER_PROC(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? KERTILE_PROC(float, avx, , fma) :
KERTILE_PROC(float, avx, _fma, fma);
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, avx, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, avx, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose);
context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, avx, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
context->mat[F64].ker_fma = is_fma ? KER_PROC(double, avx, , fma) :
KER_PROC(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? KERTILE_PROC(double, avx, , fma) :
KERTILE_PROC(double, avx, _fma, fma);
context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
@@ -117,24 +114,22 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, 512, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose);
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
context->mat[F32].ker_fma = KER_PROC(float, 512, , fma);
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, 512, , fma);
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, 512, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].pivot = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
context->mat[F64].ker_fma = KER_PROC(double, 512, , fma);
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, 512, , fma);
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
StgSharp.Native/sn-src/m.basic.x86.c
+13 -8
View File
@@ -6,10 +6,11 @@
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
// Matrix transpose implementations
SN_MK_PROC_DECL_STD(float, sse, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m128 col0 = right->f32_x[0];
register __m128 col1 = right->f32_x[1];
register __m128 col2 = right->f32_x[2];
@@ -19,14 +20,15 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose)
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
_mm_store_ps(&(ans->f32_x[0]), _mm_movelh_ps(tmp0, tmp2));
_mm_store_ps(&(ans->f32_x[1]), _mm_movehl_ps(tmp2, tmp0));
_mm_store_ps(&(ans->f32_x[2]), _mm_movelh_ps(tmp1, tmp3));
_mm_store_ps(&(ans->f32_x[3]), _mm_movehl_ps(tmp3, tmp1));
}
SN_MK_PROC_DECL_STD(float, avx, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m256 col_low = right->f32_y[0];
register __m256 col_high = right->f32_y[1];
@@ -36,18 +38,21 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose)
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
_mm256_store_ps(&(ans->f32_y[0]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)));
_mm256_store_ps(&(ans->f32_y[1]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)));
}
SN_MK_PROC_DECL_STD(float, 512, , transpose)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
register __m512 matrix = right->f32_z[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
_mm512_store_ps(&(ans->f32_z[0]), _mm512_permutexvar_ps(transpose_indices, matrix));
}
#endif
StgSharp.Native/sn-src/m.bufop.x86.c
+39
View File
@@ -20,6 +20,7 @@ SN_MK_PROC_DECL_STD(float, sse, , fill)
SN_MK_PROC_DECL_STD(float, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f32_y[0] = vec;
@@ -29,6 +30,7 @@ SN_MK_PROC_DECL_STD(float, avx, , fill)
SN_MK_PROC_DECL_STD(float, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f32_z[0] = vec;
@@ -37,6 +39,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fill)
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_x[0] = vec;
@@ -52,6 +55,7 @@ SN_MK_PROC_DECL_STD(double, sse, , fill)
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f64_y[0] = vec;
@@ -63,6 +67,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
@@ -89,6 +94,8 @@ SN_MK_PROC_DECL_STD(float, sse, , add)
SN_MK_PROC_DECL_STD(float, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -100,6 +107,8 @@ SN_MK_PROC_DECL_STD(float, avx, , add)
SN_MK_PROC_DECL_STD(float, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
@@ -107,6 +116,8 @@ SN_MK_PROC_DECL_STD(float, 512, , add)
}
SN_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
_mm_castps_pd(right[i].f32_x[0]));
@@ -138,6 +149,8 @@ SN_MK_PROC_DECL_STD(double, sse, , add)
SN_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256d c0 = _mm256_add_pd(left[i].f64_y[0], right[i].f64_y[0]);
@@ -153,6 +166,8 @@ SN_MK_PROC_DECL_STD(double, avx, , add)
SN_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
register __m512d c1 = _mm512_add_pd(left[i].f64_z[1], right[i].f64_z[1]);
@@ -163,6 +178,8 @@ SN_MK_PROC_DECL_STD(double, 512, , add)
SN_MK_PROC_DECL_STD(float, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
register __m128 c1 = _mm_sub_ps(left[i].f32_x[1], right[i].f32_x[1]);
@@ -178,6 +195,8 @@ SN_MK_PROC_DECL_STD(float, sse, , sub)
SN_MK_PROC_DECL_STD(float, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
@@ -188,6 +207,8 @@ SN_MK_PROC_DECL_STD(float, avx, , sub)
SN_MK_PROC_DECL_STD(float, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
@@ -196,6 +217,8 @@ SN_MK_PROC_DECL_STD(float, 512, , sub)
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
register __m128d c1 = _mm_sub_pd(left[i].f64_x[1], right[i].f64_x[1]);
@@ -219,6 +242,8 @@ SN_MK_PROC_DECL_STD(double, sse, , sub)
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].f32_y[1], right[i].f32_y[1]);
@@ -229,6 +254,8 @@ SN_MK_PROC_DECL_STD(double, avx, , sub)
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
@@ -237,6 +264,8 @@ SN_MK_PROC_DECL_STD(double, 512, , sub)
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_mul_ps(right[i].f32_x[0], vec);
@@ -253,6 +282,8 @@ SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].f32_y[0], vec);
@@ -265,6 +296,8 @@ SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].f32_z[0], vec);
@@ -275,6 +308,8 @@ SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_mul_pd(right[i].f64_x[0], vec);
@@ -299,6 +334,8 @@ SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
register __m256d c0 = _mm256_mul_pd(right[i].f64_y[0], vec);
@@ -315,6 +352,8 @@ SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
StgSharp.Native/sn-src/m.fma_tile.x86.c
+4 -1
View File
@@ -3,7 +3,8 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
@@ -13,6 +14,8 @@
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std();
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
int commonK = enumeration->m128_i32[2];
StgSharp.Native/sn-src/simdid_compare.x86.c
+1 -1
View File
@@ -4,7 +4,7 @@
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.context.x86.h"
/* -----------------------------------------------------------------------
* Field extraction helpers.