youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

macro gymnastics to implement storage kernel, buffer op and complex compute panel

Browse Source
This commit is contained in:
Youmu Liu
2025-11-13 19:09:27 +08:00
parent 2dfb4ae86a
commit 1417785726
18 changed files with 373 additions and 383 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.h
+163 -52
View File
@@ -33,71 +33,179 @@ typedef union uint64_u {
typedef struct mat_kernel mat_kernel;
typedef struct mat_panel mat_panel;
typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T) \
struct { \
MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */ \
MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */ \
MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \
int32_t element_size; /* Offset 24: sizeof(T) */ \
int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */ \
int32_t left_prim_offset; /* Offset 32 */ \
int32_t left_prim_stride; /* Offset 36 */ \
int32_t left_sec_offset; /* Offset 40 */ \
int32_t left_sec_stride; /* Offset 44 */ \
int32_t right_prim_offset; /* Offset 48 */ \
int32_t right_prim_stride; /* Offset 52 */ \
int32_t right_sec_offset; /* Offset 56 */ \
int32_t right_sec_stride; /* Offset 60 */ \
int32_t result_prim_offset; /* Offset 64 */ \
int32_t result_prim_stride; /* Offset 68 */ \
int32_t result_sec_offset; /* Offset 72 */ \
int32_t result_sec_stride; /* Offset 76 */ \
scalar_pack *scalar; /* Offset 80 */ \
void *compute_handle; /* Offset 88 */ \
int32_t prim_count; /* Offset 96 */ \
int32_t compute_mode; /* Offset 100 */ \
int32_t sec_count; /* Offset 104 */ \
int32_t reserved_int0; /* Offset 108 */ \
int32_t prim_tile_offset; /* Offset 112 */ \
int32_t sec_tile_offset; /* Offset 116 */ \
int32_t prim_cols_in_tile; /* Offset 120 */ \
int32_t sec_cols_in_tile; /* Offset 124 */ \
}
typedef struct matrix_task matrix_task;
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
_Static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
int count);
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
scalar_pack *scalar, int count);
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, int count)
#pragma endregion
#pragma region matrix panel function type
typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans);
typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans,
scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \
MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
#pragma region matrix kernel function type
typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar);
#define KER_PROC(T_type, T_arch, T_op_name) T_type##_kernel_##T_op_name##_##T_arch
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar)
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans)
#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans)
#pragma endregion
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef void(SN_DECL *PANEL_PROC_AS)(MAT_PANEL_SSE(int));
typedef void(SN_DECL *MAT_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
typedef void(SN_DECL *MAT_PROC_LA)(mat_kernel *left, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
typedef void(SN_DECL *MAT_PROC_LSA)(mat_kernel *matrix, scalar_pack *scalar, mat_kernel *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
#pragma region matix function
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
INTERNAL void SN_DECL f32_add_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_add_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_sse(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_avx(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_sub_512(MAT_KERNEL(float) const *left, MAT_KERNEL(float) const *right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_transpose_sse(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_avx(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_transpose_512(MAT_KERNEL(float) const *source,
MAT_KERNEL(float) *restrict target);
INTERNAL void SN_DECL f32_scalar_mul_sse(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_avx(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_scalar_mul_512(MAT_KERNEL(float) const *matrix,
MAT_KERNEL(float) *restrict ans,
scalar_pack const *scalar);
INTERNAL void SN_DECL f32_fill_sse(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_avx(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
INTERNAL void SN_DECL f32_fill_512(MAT_KERNEL(float) *restrict dest, scalar_pack *scaler);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_KER_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
#pragma endregion
#pragma region fma
INTERNAL void SN_DECL f32_fma_sse(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_avx(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
INTERNAL void SN_DECL f32_fma_512(MAT_KERNEL(float) const *restrict left,
MAT_KERNEL(float) const *restrict right,
MAT_KERNEL(float) *restrict ans);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma);
#pragma endregion
#pragma region scaler_simd
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
@@ -115,21 +223,24 @@ INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 cons
#pragma region string
typedef int(SN_DECL *HASH)(byte const *str, int length);
typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *str, uint32_t target, int length);
#pragma endregion
typedef struct sn_intrinsic {
int (*city_hash_simplify)(byte *str, int length);
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
int (*index_pair)(short const *str, uint32_t target, int length);
MAT_PROC_LRA f32_add;
MAT_PROC_AS f32_fill;
MAT_PROC_LRA f32_fma;
MAT_PROC_LRA f32_scalar_mul;
MAT_PROC_LRA f32_sub;
MAT_PROC_LA f32_transpose;
INDEX_PAIR index_pair;
KER_PROC_LRA f32_kernel_add;
KER_PROC_AS f32_kernel_fill;
PNL_PROC_LRA f32_panel_fma;
KER_PROC_LRA f32_kernel_scalar_mul;
KER_PROC_LRA f32_kernel_sub;
KER_PROC_RA f32_kernel_transpose;
VECTORNORMALIZEPROC f32_normalize_3;
} sn_intrinsic;