youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new compute kernel design, less mem read delay

Browse Source
This commit is contained in:
Nitload
2026-04-07 01:04:05 +08:00
parent d702b3224c
commit 12a216d404
47 changed files with 225 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/m.bufop.x86.c
+2 -2
View File
@@ -68,7 +68,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
ans[i].f64_z[1] = vec;
@@ -354,7 +354,7 @@ SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
register __m512d vec = _mm512_set1_pd(((double const *)scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
StgSharp.Native/sn-src/m.solve.x86.c
+1
View File
@@ -4,6 +4,7 @@
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#include <math.h>
static const float quality_threshold = 1e-6f;
StgSharp.Native/sn-src/mk.fma.x86.c
+38 -23
View File
@@ -136,40 +136,55 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
#define SN_MK_FMA_F32_512_PHASE(i)\
do { \
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_0 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_1 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_2 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
ak = _mm512_broadcast_f32x4(left_##i ->f32_x[0]);\
c_3 = _mm512_fmadd_ps(ak, _mm512_permute_ps(b_##i, _FMA_SELECT(0)), c_##i);\
}while(0)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps((float *)ans_cur);
register __m512 c_1 = c;
register __m512 c_2 = c;
register __m512 c_3 = c;
register __m512 c_0 = _mm512_load_ps((float *)ans_cur);
register __m512 c_1 = c_0;
register __m512 c_2 = c_0;
register __m512 c_3 = c_0;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
for (size_t z = 0; z < common_z; z+=4) {
SN_MAT_KERNEL_LOCAL const *right = right + z;
SN_MAT_KERNEL_LOCAL const *left_0 = left + z + 0;
SN_MAT_KERNEL_LOCAL const *left_1 = left + z + 1;
SN_MAT_KERNEL_LOCAL const *left_2 = left + z + 2;
SN_MAT_KERNEL_LOCAL const *left_3 = left + z + 3;
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right_cur);
register __m512 b_0 = _mm512_load_ps((float *)(right+0));
register __m512 b_1 = _mm512_load_ps((float *)(right+1));
register __m512 b_2 = _mm512_load_ps((float *)(right+2));
register __m512 b_3 = _mm512_load_ps((float *)(right+3));
// k=0: broadcast ka_col0 到4个lane，permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left_cur->f32_x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left_cur->f32_x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left_cur->f32_x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left_cur->f32_x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
register __m512 ak_1;
register __m512 ak_2;
register __m512 ak_3;
SN_MK_FMA_F32_512_PHASE(0);
SN_MK_FMA_F32_512_PHASE(1);
SN_MK_FMA_F32_512_PHASE(2);
SN_MK_FMA_F32_512_PHASE(3);
}
_mm512_store_ps((float *)ans_cur, c);
c_0 = _mm512_add_ps(c_0, c_1);
c_2= _mm512_add_ps(c_2, c_3);
c_0 = _mm512_add_ps(c_0, c_2);
_mm512_store_ps((float *)ans_cur, c_0);
}
// double
StgSharp.Native/sn-src/s.hash.x86.c
+2 -3
View File
@@ -2,7 +2,6 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "StgSharpNative.h"
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
@@ -17,7 +16,7 @@ INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
}
} else if (size < 16) {
for (; ptr < end; ptr += 4) {
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr++) {
hash = _mm_crc32_u16(hash, *ptr);
@@ -34,7 +33,7 @@ INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
hash = _mm_crc32_u8(hash, hash64);
hash = _mm_crc32_u8(hash, hash64 >> 32);
for (; ptr < last_32; ptr += 4) {
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u8(hash, *ptr);