youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better L4 and array storage design

Browse Source
This commit is contained in:
Youmu Liu
2026-03-04 18:12:58 +08:00
parent 7e970352ab
commit dfa6f4ba92
103 changed files with 853 additions and 6837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/StgSharp.Native.vcxproj
-1
View File
@@ -185,7 +185,6 @@
<ClCompile Include="sn-src\simdid.x86.c" />
<ClCompile Include="sn-src\simdid_compare.x86.c" />
<ClCompile Include="sn-src\sn_native.c" />
<ClCompile Include="sn-src\str_string.c" />
<ClCompile Include="sn-src\t.affinity.c" />
<ClCompile Include="sn-src\vector.c" />
<ClCompile Include="sn-src\vector_dot.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
-3
View File
@@ -27,9 +27,6 @@
<ClCompile Include="sn-src\s.hash.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\str_string.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\vector.c">
<Filter>源文件</Filter>
</ClCompile>
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+45 -8
View File
@@ -182,15 +182,52 @@ SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , tiled_fma);
SN_MK_PROC_DECL_STD(float, avx, , tiled_fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma);
SN_MK_PROC_DECL_STD(float, 512, , tiled_fma);
SN_SK_PROC_DECL_STD(float, avx, , add);
SN_SK_PROC_DECL_STD(float, 512, , add);
SN_MK_PROC_DECL_STD(double, sse, , tiled_fma);
SN_MK_PROC_DECL_STD(double, avx, , tiled_fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma);
SN_MK_PROC_DECL_STD(double, 512, , tiled_fma);
SN_SK_PROC_DECL_STD(double, sse, , add);
SN_SK_PROC_DECL_STD(double, avx, , add);
SN_SK_PROC_DECL_STD(double, 512, , add);
SN_SK_PROC_DECL_STD(float, sse, , sub);
SN_SK_PROC_DECL_STD(float, avx, , sub);
SN_SK_PROC_DECL_STD(float, 512, , sub);
SN_SK_PROC_DECL_STD(double, sse, , sub);
SN_SK_PROC_DECL_STD(double, avx, , sub);
SN_SK_PROC_DECL_STD(double, 512, , sub);
SN_SK_PROC_DECL_STD(float, sse, , transpose);
SN_SK_PROC_DECL_STD(float, avx, , transpose);
SN_SK_PROC_DECL_STD(float, 512, , transpose);
SN_SK_PROC_DECL_STD(float, sse, , scalar_mul);
SN_SK_PROC_DECL_STD(float, avx, , scalar_mul);
SN_SK_PROC_DECL_STD(float, 512, , scalar_mul);
SN_SK_PROC_DECL_STD(double, sse, , scalar_mul);
SN_SK_PROC_DECL_STD(double, avx, , scalar_mul);
SN_SK_PROC_DECL_STD(double, 512, , scalar_mul);
SN_SK_PROC_DECL_STD(float, sse, , fill);
SN_SK_PROC_DECL_STD(float, avx, , fill);
SN_SK_PROC_DECL_STD(float, 512, , fill);
SN_SK_PROC_DECL_STD(double, sse, , fill);
SN_SK_PROC_DECL_STD(double, avx, , fill);
SN_SK_PROC_DECL_STD(double, 512, , fill);
SN_SK_PROC_DECL_STD(float, sse, , quality);
SN_SK_PROC_DECL_STD(float, sse, , fma);
SN_SK_PROC_DECL_STD(float, avx, , fma);
SN_SK_PROC_DECL_STD(float, avx, _fma, fma);
SN_SK_PROC_DECL_STD(float, 512, , fma);
SN_SK_PROC_DECL_STD(double, sse, , fma);
SN_SK_PROC_DECL_STD(double, avx, , fma);
SN_SK_PROC_DECL_STD(double, avx, _fma, fma);
SN_SK_PROC_DECL_STD(double, 512, , fma);
#pragma endregion
StgSharp.Native/sn-head/sn_intrinsic.h
+21 -11
View File
@@ -73,6 +73,8 @@ static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size
#endif
typedef void(SN_DECL *UNI_MK_PROC)(matrix_task *task);
typedef void(SN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
mat_kernel *restrict ans, uint64_t scalar);
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
@@ -85,25 +87,33 @@ typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length
#pragma endregion
typedef struct mat_intrinsic {
UNI_MK_PROC buffer_add;
UNI_MK_PROC buffer_fill;
UNI_MK_PROC buffer_scalar_mul;
UNI_MK_PROC buffer_sub;
UNI_MK_PROC buffer_transpose;
UNI_MK_PROC ker_fma;
UNI_MK_PROC ker_tile_fma;
typedef struct mat_mk_intrinsic {
UNI_MK_PROC add;
UNI_MK_PROC fill;
UNI_MK_PROC scalar_mul;
UNI_MK_PROC sub;
UNI_MK_PROC transpose;
UNI_MK_PROC fma;
UNI_MK_PROC pivot;
} mat_mk_intrinsic;
} mat_intrinsic;
typedef struct mat_sk_intrinsic {
UNI_SK_PROC add;
UNI_SK_PROC fill;
UNI_SK_PROC scalar_mul;
UNI_SK_PROC sub;
UNI_SK_PROC transpose;
UNI_SK_PROC fma;
UNI_SK_PROC pivot;
} mat_sk_intrinsic;
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
typedef struct sn_intrinsic {
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
INDEX_PAIR index_pair;
VECTORNORMALIZEPROC vec_f32_normalize_3;
mat_intrinsic mat[4];
mat_mk_intrinsic mat_mk[0x0a];
mat_sk_intrinsic mat_sk[0x0a];
} sn_intrinsic;
typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask;
StgSharp.Native/sn-head/sn_intrinsic.std.h
+9 -3
View File
@@ -10,12 +10,18 @@
#define mat_task_local ((_mat_task_local const *)task)
#define STD_MK_PROC_NAME(T_type, T_arch, T_feature, T_op) \
T_type##_mat_proc_##T_op##_##T_arch##_##T_feature
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL STD_MK_PROC_NAME(T_type, T_arch, T_feature, \
INTERNAL void SN_DECL MK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) const *task)
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
#define _SN_STD__PROBE_0 0
#define _SN_STD__PROBE_1 1
StgSharp.Native/sn-src/load_intrinsic.x86.c
+44 -55
View File
@@ -50,25 +50,22 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, sse, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, sse, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, sse, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, sse, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, sse, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, sse, , transpose);
context->mat_mk[F32].fma = MK_PROC_NAME_STD(float, sse, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, sse, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, sse, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, sse, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, sse, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = MK_PROC_NAME_STD(double, sse, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -81,29 +78,24 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, avx, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, avx, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose);
context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) :
STD_MK_PROC_NAME(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , tiled_fma) :
STD_MK_PROC_NAME(float, avx, _fma, tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, avx, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, avx, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, avx, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, avx, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, avx, , transpose);
context->mat_mk[F32].fma = is_fma ? MK_PROC_NAME_STD(float, avx, , fma) :
MK_PROC_NAME_STD(float, avx, _fma, fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, avx, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) :
STD_MK_PROC_NAME(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , tiled_fma) :
STD_MK_PROC_NAME(double, avx, _fma, tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, avx, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, avx, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, avx, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, avx, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = is_fma ? MK_PROC_NAME_STD(double, avx, , fma) :
MK_PROC_NAME_STD(double, avx, _fma, fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -112,25 +104,22 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = STD_MK_PROC_NAME(float, 512, , add);
context->mat[F32].buffer_fill = STD_MK_PROC_NAME(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = STD_MK_PROC_NAME(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub);
context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose);
context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma);
context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , tiled_fma);
context->mat[F32].pivot = NULL;
context->mat_mk[F32].add = MK_PROC_NAME_STD(float, 512, , add);
context->mat_mk[F32].fill = MK_PROC_NAME_STD(float, 512, , fill);
context->mat_mk[F32].scalar_mul = MK_PROC_NAME_STD(float, 512, , scalar_mul);
context->mat_mk[F32].sub = MK_PROC_NAME_STD(float, 512, , sub);
context->mat_mk[F32].transpose = MK_PROC_NAME_STD(float, 512, , transpose);
context->mat_mk[F32].fma = MK_PROC_NAME_STD(float, 512, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add);
context->mat[F64].buffer_fill = STD_MK_PROC_NAME(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = STD_MK_PROC_NAME(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma);
context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , tiled_fma);
context->mat[F64].pivot = NULL;
context->mat_mk[F64].add = MK_PROC_NAME_STD(double, 512, , add);
context->mat_mk[F64].fill = MK_PROC_NAME_STD(double, 512, , fill);
context->mat_mk[F64].scalar_mul = MK_PROC_NAME_STD(double, 512, , scalar_mul);
context->mat_mk[F64].sub = MK_PROC_NAME_STD(double, 512, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = MK_PROC_NAME_STD(double, 512, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
StgSharp.Native/sn-src/m.bufop.x86.c
+9 -9
View File
@@ -39,7 +39,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fill)
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_x[0] = vec;
@@ -55,7 +55,7 @@ SN_MK_PROC_DECL_STD(double, sse, , fill)
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
ans[i].f64_y[0] = vec;
@@ -67,7 +67,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
@@ -116,7 +116,7 @@ SN_MK_PROC_DECL_STD(float, 512, , add)
}
SN_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
@@ -149,7 +149,7 @@ SN_MK_PROC_DECL_STD(double, sse, , add)
SN_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -166,7 +166,7 @@ SN_MK_PROC_DECL_STD(double, avx, , add)
SN_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
@@ -217,7 +217,7 @@ SN_MK_PROC_DECL_STD(float, 512, , sub)
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
@@ -242,7 +242,7 @@ SN_MK_PROC_DECL_STD(double, sse, , sub)
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -254,7 +254,7 @@ SN_MK_PROC_DECL_STD(double, avx, , sub)
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
StgSharp.Native/sn-src/m.fma.x86.c
+8 -24
View File
@@ -10,10 +10,8 @@
// float
SN_MK_PROC_DECL_STD(float, sse, , fma)
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
@@ -60,10 +58,8 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
_mm_store_ps((float *)&ans->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -108,10 +104,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
ans->f32_y[1] = c23;
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -156,10 +150,8 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
ans->f32_y[1] = c23;
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
@@ -194,10 +186,8 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
// double
SN_MK_PROC_DECL_STD(double, sse, , fma)
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
@@ -240,10 +230,8 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
}
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -286,10 +274,8 @@ SN_MK_PROC_DECL_STD(double, avx, , fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -332,10 +318,8 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/m.fma_tile.x86.c
+8 -8
View File
@@ -12,7 +12,7 @@
// float
SN_MK_PROC_DECL_STD(float, sse, , tiled_fma)
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
@@ -69,7 +69,7 @@ SN_MK_PROC_DECL_STD(float, sse, , tiled_fma)
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , tiled_fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -127,7 +127,7 @@ SN_MK_PROC_DECL_STD(float, avx, , tiled_fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -185,7 +185,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
SN_MK_PROC_DECL_STD(float, 512, , tiled_fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
@@ -260,7 +260,7 @@ SN_MK_PROC_DECL_STD(float, 512, , tiled_fma)
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
SN_MK_PROC_DECL_STD(double, sse, , tiled_fma)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -299,7 +299,7 @@ SN_MK_PROC_DECL_STD(double, sse, , tiled_fma)
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
}
SN_MK_PROC_DECL_STD(double, avx, , tiled_fma)
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -372,7 +372,7 @@ SN_MK_PROC_DECL_STD(double, avx, , tiled_fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
@@ -429,7 +429,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , tiled_fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/str_string.c
-54
View File
@@ -1,54 +0,0 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length)
{
int index = 0, result = 0, size = length, end = size - 8;
__m128i s = _mm_set1_epi32(target);
__m128i vec, com;
while (index < end) {
vec = _mm_loadu_epi32(str + index);
com = _mm_cmpeq_epi16(vec, s);
result = _mm_movemask_epi8(com);
if (result) {
for (size_t i = 0; i < 7; i++) {
if (((result & 0xF) == 0xF)) {
return index + i;
}
if (result == 0) {
break;
}
result >>= 2;
}
}
index += 1;
vec = _mm_loadu_epi32(str + index);
com = _mm_cmpeq_epi16(vec, s);
result = _mm_movemask_epi8(com);
if (result) {
for (size_t i = 0; i < 7; i++) {
if (((result & 0xF) == 0xF)) {
return index + i;
}
if (result == 0) {
break;
}
result >>= 2;
}
}
index += 7;
}
for (; (index < size - 1) && (!result); index += 2) {
result = *(int *)(str + index) == target;
}
if (result) {
return index;
}
return -1;
}
#endif