youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better L4 and array storage design

Browse Source
This commit is contained in:
Youmu Liu
2026-03-04 18:12:58 +08:00
parent 7e970352ab
commit dfa6f4ba92
103 changed files with 853 additions and 6837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/load_intrinsic.x86.c
+44 -55
View File
@@ -50,25 +50,22 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, sse, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, sse, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, sse, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, sse, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, sse, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, sse, , transpose);
context->mat_mk[F32].fma = MK_PROC_NAME_STD(float, sse, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, sse, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, sse, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, sse, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, sse, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = MK_PROC_NAME_STD(double, sse, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -81,29 +78,24 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, avx, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, avx, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose);
context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , tiled_fma) :
STD_MK_PROC_NAME(float, avx, _fma, tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, avx, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, avx, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, avx, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, avx, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, avx, , transpose);
context->mat_mk[F32].fma = is_fma ? MK_PROC_NAME_STD(float, avx, , fma) :
MK_PROC_NAME_STD(float, avx, _fma, fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, avx, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , tiled_fma) :
STD_MK_PROC_NAME(double, avx, _fma, tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, avx, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, avx, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, avx, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, avx, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = is_fma ? MK_PROC_NAME_STD(double, avx, , fma) :
MK_PROC_NAME_STD(double, avx, _fma, fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -112,25 +104,22 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, 512, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, 512, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, 512, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, 512, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, 512, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, 512, , transpose);
context->mat_mk[F32].fma = MK_PROC_NAME_STD(float, 512, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, 512, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, 512, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, 512, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, 512, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = MK_PROC_NAME_STD(double, 512, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
StgSharp.Native/sn-src/m.bufop.x86.c
+9 -9
View File
@@ -39,7 +39,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fill)
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_x[0] = vec;
@@ -55,7 +55,7 @@ SN_MK_PROC_DECL_STD(double, sse, , fill)
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f64_y[0] = vec;
@@ -67,7 +67,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
@@ -116,7 +116,7 @@ SN_MK_PROC_DECL_STD(float, 512, , add)
}
SN_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
@@ -149,7 +149,7 @@ SN_MK_PROC_DECL_STD(double, sse, , add)
SN_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -166,7 +166,7 @@ SN_MK_PROC_DECL_STD(double, avx, , add)
SN_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
@@ -217,7 +217,7 @@ SN_MK_PROC_DECL_STD(float, 512, , sub)
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
@@ -242,7 +242,7 @@ SN_MK_PROC_DECL_STD(double, sse, , sub)
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -254,7 +254,7 @@ SN_MK_PROC_DECL_STD(double, avx, , sub)
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
StgSharp.Native/sn-src/m.fma.x86.c
+8 -24
View File
@@ -10,10 +10,8 @@
// float
SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
@@ -60,10 +58,8 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
_mm_store_ps((float *)&ans->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -108,10 +104,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
ans->f32_y[1] = c23;
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -156,10 +150,8 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
ans->f32_y[1] = c23;
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
@@ -194,10 +186,8 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
// double
SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
@@ -240,10 +230,8 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
}
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -286,10 +274,8 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -332,10 +318,8 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/m.fma_tile.x86.c
+8 -8
View File
@@ -12,7 +12,7 @@
// float
SN_MK_PROC_DECL_STD(float, sse, , tiled_fma)
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
@@ -69,7 +69,7 @@ SN_MK_PROC_DECL_STD(float, sse, , tiled_fma)
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , tiled_fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -127,7 +127,7 @@ SN_MK_PROC_DECL_STD(float, avx, , tiled_fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -185,7 +185,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, 512, , tiled_fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
@@ -260,7 +260,7 @@ SN_MK_PROC_DECL_STD(float, 512, , tiled_fma)
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
SN_MK_PROC_DECL_STD(double, sse, , tiled_fma)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -299,7 +299,7 @@ SN_MK_PROC_DECL_STD(double, sse, , tiled_fma)
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
}
SN_MK_PROC_DECL_STD(double, avx, , tiled_fma)
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -372,7 +372,7 @@ SN_MK_PROC_DECL_STD(double, avx, , tiled_fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -429,7 +429,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , tiled_fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/str_string.c
-54
View File
@@ -1,54 +0,0 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
{
int index = 0, result = 0, size = length, end = size - 8;
__m128i s = _mm_set1_epi32(target);
__m128i vec, com;
while (index < end) {
vec = _mm_loadu_epi32(str + index);
com = _mm_cmpeq_epi16(vec, s);
result = _mm_movemask_epi8(com);
if (result) {
for (size_t i = 0; i < 7; i++) {
if (((result & 0xF) == 0xF)) {
return index + i;
}
if (result == 0) {
break;
}
result >>= 2;
}
}
index += 1;
vec = _mm_loadu_epi32(str + index);
com = _mm_cmpeq_epi16(vec, s);
result = _mm_movemask_epi8(com);
if (result) {
for (size_t i = 0; i < 7; i++) {
if (((result & 0xF) == 0xF)) {
return index + i;
}
if (result == 0) {
break;
}
result >>= 2;
}
}
index += 7;
}
for (; (index < size - 1) && (!result); index += 2) {
result = *(int *)(str + index) == target;
}
if (result) {
return index;
}
return -1;
}
#endif