youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

arrange of proj structure

Browse Source
This commit is contained in:
Youmu Liu
2026-02-27 17:09:13 +08:00
parent 15434ed053
commit a798ab07f1
173 changed files with 32304 additions and 33955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/StgSharp.Native.vcxproj
-1
View File
@@ -155,7 +155,6 @@
</Link>
</ItemDefinitionGroup>
<ItemGroup>
<ClCompile Include="sn-src\m.panel.x86.c" />
<ClCompile Include="sn-src\m.basic.x86.c" />
<ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\m.bufop.x86.c" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
-3
View File
@@ -54,9 +54,6 @@
<ClCompile Include="sn-src\m.bufop.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.panel.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.solve.x86.c">
<Filter>源文件</Filter>
</ClCompile>
StgSharp.Native/sn-head/sn_intrinsic.h
+28 -60
View File
@@ -59,41 +59,22 @@ typedef union scalar_pack {
#define LEFT_RIGHT_ANS_SCALAR_PARAM 4
#define ANS_SCALAR_PARAM 5
typedef struct matrix_op_mode {
int16_t operation_style; /* PANEL_OP, BUFFER_OP, KERNEL_OP */
int16_t parameter_style; /* RIGHT_ANS_PARAM, LEFT_RIGHT_ANS_PARAM, e.g. */
} matrix_op_mode;
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
#define MAT_TASK(T) \
struct { \
MAT_KERNEL(T) * left; /* Offset 0: left matrix kernel base */ \
MAT_KERNEL(T) * right; /* Offset 8: right matrix kernel base */ \
MAT_KERNEL(T) * result; /* Offset 16: result matrix kernel base */ \
int32_t element_size; /* Offset 24: sizeof(T) */ \
int32_t reserved_ptr0; /* Offset 28: reserved/extension slot */ \
int32_t left_prim_offset; /* Offset 32 */ \
int32_t left_prim_stride; /* Offset 36 */ \
int32_t left_sec_offset; /* Offset 40 */ \
int32_t left_sec_stride; /* Offset 44 */ \
int32_t right_prim_offset; /* Offset 48 */ \
int32_t right_prim_stride; /* Offset 52 */ \
int32_t right_sec_offset; /* Offset 56 */ \
int32_t right_sec_stride; /* Offset 60 */ \
int32_t result_prim_offset; /* Offset 64 */ \
int32_t result_prim_stride; /* Offset 68 */ \
int32_t result_sec_offset; /* Offset 72 */ \
int32_t result_sec_stride; /* Offset 76 */ \
scalar_pack *scalar; /* Offset 80 */ \
void *compute_handle; /* Offset 88 */ \
int32_t prim_count; /* Offset 96 */ \
matrix_op_mode compute_mode; /* Offset 100 */ \
int32_t sec_count; /* Offset 104 */ \
int32_t reserved_int0; /* Offset 108 */ \
int32_t prim_tile_offset; /* Offset 112 */ \
int32_t sec_tile_offset; /* Offset 116 */ \
int32_t prim_cols_in_tile; /* Offset 120 */ \
int32_t sec_cols_in_tile; /* Offset 124 */ \
#define MAT_TASK(T) \
struct { \
MAT_KERNEL(T) * mat_0; /* Offset 0: 1st matrix kernel base */ \
MAT_KERNEL(T) * mat_1; /* Offset 8: 2nd matrix kernel base */ \
MAT_KERNEL(T) * mat_2; /* Offset 16: 3rd matrix kernel base */ \
void *scalar; /* Offset 24: scalar pack */ \
int32_t i; /* Offset 32: row index of the result tile */ \
int32_t j; /* Offset 36: column index of the result tile */ \
int32_t k; /* Offset 40: column index of the result tile */ \
int32_t common_i; /* Offset 44: row index of the result tile */ \
int32_t common_j; /* Offset 48: column index of the result tile */ \
int32_t common_k; /* Offset 52: column index of the result tile */ \
uint32_t mask; /* Offset 56: mask for task schedualing, not useful in C side */ \
uint32_t reverse; /* Offset 60: reserved for future use */ \
scalar_pack padding; /* Offset 64: padding to make the struct size 128 bytes */ \
}
typedef struct matrix_task matrix_task;
@@ -103,6 +84,9 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
typedef void(SN_DECL *UNIFIKERNELPROC)(matrix_task *task);
#define UNIFIKERNRLPROC(T_arch, T_feature, T_op) mat_proc_##T_op##_##T_arch##_##T_feature
#pragma region matrix buffer operation function type
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
@@ -247,17 +231,15 @@ typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length
#pragma endregion
typedef struct mat_intrinsic {
BUF_PROC_LRA buffer_add;
BUF_PROC_AS buffer_fill;
BUF_PROC_RAS buffer_scalar_mul;
BUF_PROC_LRA buffer_sub;
KER_PROC_RA buffer_transpose;
PANEL_PROC build_panel;
PANEL_PROC clear_panel;
PNL_PROC_LRA ker_fma;
KERTILE_PROC_LRA ker_tile_fma;
PIVOTPROC pivot;
PANEL_PROC store_panel;
UNIFIKERNELPROC buffer_add;
UNIFIKERNELPROC buffer_fill;
UNIFIKERNELPROC buffer_scalar_mul;
UNIFIKERNELPROC buffer_sub;
UNIFIKERNELPROC buffer_transpose;
UNIFIKERNELPROC clear_panel;
UNIFIKERNELPROC ker_fma;
UNIFIKERNELPROC ker_tile_fma;
UNIFIKERNELPROC pivot;
} mat_intrinsic;
@@ -270,21 +252,7 @@ typedef struct sn_intrinsic {
mat_intrinsic mat[4];
} sn_intrinsic;
typedef enum elment_type { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_type;
typedef enum most_advanced_instruction {
#ifdef _MSC_VER
SSE = __IA_SUPPORT_SSE42,
AVX256 = __IA_SUPPORT_VECTOR256,
AVX512 = __IA_SUPPORT_VECTOR512,
NEON
#else
SSE,
AVX256,
AVX512,
NEON
#endif
} most_advanced_instruction;
typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask;
#include "sn_target.h"
StgSharp.Native/sn-head/sn_matkernel.h
+2 -1
View File
@@ -125,7 +125,7 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
int col_length, int row_length, \
int col_index, int row_index)
/* Declarations for AVX float variant */
/* Declarations for AVX float variant
DECLARE_BUILD_PANEL(avx, float);
DECLARE_BUILD_PANEL(512, float);
@@ -136,5 +136,6 @@ DECLARE_STORE_PANEL(512, float);
DECLARE_BUILD_PANEL(512, double);
DECLARE_STORE_PANEL(512, double);
/**/
#endif /* SN_MATKERNEL */
StgSharp.Native/sn-head/sn_task_expand.h
View File
StgSharp.Native/sn-src/load_intrinsic.x86.c
-12
View File
@@ -56,24 +56,20 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, , transpose);
context->mat[F32].build_panel = NULL;
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
context->mat[F32].ker_fma = KER_PROC(float, sse, , fma);
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, sse, , fma);
context->mat[F32].pivot = NULL;
context->mat[F32].store_panel = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
context->mat[F64].ker_fma = KER_PROC(double, sse, , fma);
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, sse, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -93,28 +89,24 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, , transpose);
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
context->mat[F32].ker_fma = is_fma ? KER_PROC(float, avx, , fma) :
KER_PROC(float, avx, _fma, fma);
context->mat[F32].ker_tile_fma = is_fma ? KERTILE_PROC(float, avx, , fma) :
KERTILE_PROC(float, avx, _fma, fma);
context->mat[F32].pivot = NULL;
context->mat[F32].store_panel = STORE_PANEL(avx, float);
context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
context->mat[F64].ker_fma = is_fma ? KER_PROC(double, avx, , fma) :
KER_PROC(double, avx, _fma, fma);
context->mat[F64].ker_tile_fma = is_fma ? KERTILE_PROC(double, avx, , fma) :
KERTILE_PROC(double, avx, _fma, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
@@ -130,24 +122,20 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, , transpose);
context->mat[F32].build_panel = BUILD_PANEL(512, float);
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
context->mat[F32].ker_fma = KER_PROC(float, 512, , fma);
context->mat[F32].ker_tile_fma = KERTILE_PROC(float, 512, , fma);
context->mat[F32].pivot = NULL;
context->mat[F32].store_panel = STORE_PANEL(512, float);
context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = BUILD_PANEL(512, double);
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
context->mat[F64].ker_fma = KER_PROC(double, 512, , fma);
context->mat[F64].ker_tile_fma = KERTILE_PROC(double, 512, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = STORE_PANEL(512, double);
context->vec_f32_normalize_3 = f32_normalize;
}
StgSharp.Native/sn-src/m.fma.x86.c
+18 -18
View File
@@ -244,33 +244,33 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_load_pd((double *)&ans->f64_y[0]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3))));
ans->f64_y[0] = ccol;
_mm256_store_pd((double *)&ans->f64_y[0], ccol);
/* Column 1 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_load_pd((double *)&ans->f64_y[1]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3))));
ans->f64_y[1] = ccol;
_mm256_store_pd((double *)&ans->f64_y[1], ccol);
/* Column 2 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_load_pd((double *)&ans->f64_y[2]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3))));
ans->f64_y[2] = ccol;
_mm256_store_pd((double *)&ans->f64_y[2], ccol);
/* Column 3 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_load_pd((double *)&ans->f64_y[3]);
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2))));
ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3))));
ans->f64_y[3] = ccol;
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
@@ -288,33 +288,33 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
register __m256d ccol;
/* Column 0 */
ccol = _mm256_load_pd((double *)&ans->f32_y[0]);
ccol = _mm256_load_pd((double *)&ans->f64_y[0]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol);
ans->f64_y[0] = ccol;
_mm256_store_pd((double *)&ans->f64_y[0], ccol);
/* Column 1 */
ccol = _mm256_load_pd((double *)&ans->f32_y[1]);
ccol = _mm256_load_pd((double *)&ans->f64_y[1]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)), ccol);
ans->f64_y[1] = ccol;
_mm256_store_pd((double *)&ans->f64_y[1], ccol);
/* Column 2 */
ccol = _mm256_load_pd((double *)&ans->f32_y[2]);
ccol = _mm256_load_pd((double *)&ans->f64_y[2]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)), ccol);
ans->f64_y[2] = ccol;
_mm256_store_pd((double *)&ans->f64_y[2], ccol);
/* Column 3 */
ccol = _mm256_load_pd((double *)&ans->f32_y[3]);
ccol = _mm256_load_pd((double *)&ans->f64_y[3]);
ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)), ccol);
ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)), ccol);
ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)), ccol);
ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)), ccol);
ans->f64_y[3] = ccol;
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
@@ -359,8 +359,8 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
c01 = _mm512_fmadd_pd(a30, b01_perm, c01);
c23 = _mm512_fmadd_pd(a12, b23_perm, c23);
ans->f64_z[0] = c01;
ans->f64_z[1] = c23;
_mm512_store_pd((double *)&ans->f64_z[0], c01);
_mm512_store_pd((double *)&ans->f64_z[1], c23);
}
#endif
StgSharp.Native/sn-src/m.fma_tile.x86.c
+5 -5
View File
@@ -277,7 +277,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0);
} while (0)
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
@@ -300,13 +300,13 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
const *left_cur = left + GET_INDEX(ansHeight, i, k),
const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe
DOUBLE_KERNEL_FMA_CYCLE(0)
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1)
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2)
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3)
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);