youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mat proc kernel std tamplate is ok, merging old decl

Browse Source
This commit is contained in:
Nitload
2026-03-02 00:24:15 +08:00
parent e198f22712
commit 67fbbde449
8 changed files with 279 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+70 -77
View File
@@ -3,6 +3,7 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.x86.md)
@@ -57,39 +58,39 @@
#define SIMDID_MAIN_LVL_AVX10 0x05u
#ifndef SIMDID_AVX512_BASE
#define SIMDID_AVX512_FP16 U << 13) relative bit 13 within [31..16] */
#define SIMDID_AVX512_BF16 U << 12)
#define SIMDID_AVX512_VNNI U << 11)
#define SIMDID_AVX512_VBMI2 U << 10)
#define SIMDID_AVX512_VBMI U << 9)
#define SIMDID_AVX512_AI_RESERVED U << 8)
#define SIMDID_AVX512_BASE U << 4)
#define SIMDID_AVX512_IMPL_SHIFT
#define SIMDID_AVX512_IMPL_MASK x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE x2U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_FP16 (1U << 13) /* relative bit 13 within [31..16] */
#define SIMDID_AVX512_BF16 (1U << 12)
#define SIMDID_AVX512_VNNI (1U << 11)
#define SIMDID_AVX512_VBMI2 (1U << 10)
#define SIMDID_AVX512_VBMI (1U << 9)
#define SIMDID_AVX512_AI_RESERVED (1U << 8)
#define SIMDID_AVX512_BASE (1U << 4)
#define SIMDID_AVX512_IMPL_SHIFT 6
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
#endif
#ifndef SIMDID_AVX_BASE
#define SIMDID_AVX U << 0) relative bit 0 within [47..40] */
#define SIMDID_AVX2 U << 1)
#define SIMDID_AVX_FMA U << 2)
#define SIMDID_AVX (1U << 0) /* relative bit 0 within [47..40] */
#define SIMDID_AVX2 (1U << 1)
#define SIMDID_AVX_FMA (1U << 2)
#define SIMDID_AVX_F16C (1U << 3)
#define SIMDID_AVX_DUAL ( \
U << 4) glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
#define SIMDID_AVX_DUAL \
(1U << 4) /* glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
#endif
#ifndef SIMDID_AMX_TILE
#define SIMDID_AMX_TILE (1U << 0) relative bit 0 within [39..32] */
#define SIMDID_AMX_TILE (1U << 0) /* relative bit 0 within [39..32] */
#define SIMDID_AMX_INT8 (1U << 1)
#define SIMDID_AMX_BF16 (1U << 2)
#endif
#ifndef SIMDID_AVX10_BASE
#define SIMDID_AVX10 U << 0) /* relative bit 0 within [55..48] */
#define SIMDID_AVX10_V1 U << 1)
#define SIMDID_AVX10_V2 U << 2)
#define SIMDID_AVX10 (1U << 0) /* relative bit 0 within [55..48] */
#define SIMDID_AVX10_V1 (1U << 1)
#define SIMDID_AVX10_V2 (1U << 2)
#define SIMDID_AVX10_256W (1U << 3)
#define SIMDID_AVX10_512W (1U << 4)
#endif
@@ -102,10 +103,10 @@
* 10 = P-core (performance/big core in a hybrid CPU)
* 11 = unified (confirmed non-hybrid; numerically strongest)
* Higher numeric value == stronger / higher-capability topology. */
#define SIMDID_UARCH_HYBRID_E << 0) / E-core (efficiency/small) in a hybrid topology */
#define SIMDID_UARCH_HYBRID_P << 1) / P-core (performance/big) in a hybrid topology */
#define SIMDID_UARCH_SIMD_DUAL ( \
<< 2) / SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
#define SIMDID_UARCH_HYBRID_E (1U << 0) /* E-core (efficiency/small) in a hybrid topology */
#define SIMDID_UARCH_HYBRID_P (1U << 1) /* P-core (performance/big) in a hybrid topology */
#define SIMDID_UARCH_SIMD_DUAL \
(1U << 2) /* SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
#endif
// Pack helpers (define locally to avoid missing macros when included standalone)
@@ -133,71 +134,63 @@
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
STD_MK_PROC_DECL(float, sse, , add);
STD_MK_PROC_DECL(float, avx, , add);
STD_MK_PROC_DECL(float, 512, , add);
SN_MK_PROC_DECL_STD(float, sse, , add);
SN_MK_PROC_DECL_STD(float, avx, , add);
SN_MK_PROC_DECL_STD(float, 512, , add);
STD_MK_PROC_DECL(double, sse, , add);
STD_MK_PROC_DECL(double, avx, , add);
STD_MK_PROC_DECL(double, 512, , add);
SN_MK_PROC_DECL_STD(double, sse, , add);
SN_MK_PROC_DECL_STD(double, avx, , add);
SN_MK_PROC_DECL_STD(double, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
SN_MK_PROC_DECL_STD(float, sse, , sub);
SN_MK_PROC_DECL_STD(float, avx, , sub);
SN_MK_PROC_DECL_STD(float, 512, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
SN_MK_PROC_DECL_STD(double, sse, , sub);
SN_MK_PROC_DECL_STD(double, avx, , sub);
SN_MK_PROC_DECL_STD(double, 512, , sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose);
SN_MK_PROC_DECL_STD(float, sse, , transpose);
SN_MK_PROC_DECL_STD(float, avx, , transpose);
SN_MK_PROC_DECL_STD(float, 512, , transpose);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
SN_MK_PROC_DECL_STD(float, sse, , fill);
SN_MK_PROC_DECL_STD(float, avx, , fill);
SN_MK_PROC_DECL_STD(float, 512, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
SN_MK_PROC_DECL_STD(double, sse, , fill);
SN_MK_PROC_DECL_STD(double, avx, , fill);
SN_MK_PROC_DECL_STD(double, 512, , fill);
DECLARE_PNL_PROC_ANS(float, sse, , clear);
DECLARE_PNL_PROC_ANS(float, avx, , clear);
DECLARE_PNL_PROC_ANS(float, 512, , clear);
SN_MK_PROC_DECL_STD(float, sse, , quality);
DECLARE_PNL_PROC_ANS(double, sse, , clear);
DECLARE_PNL_PROC_ANS(double, avx, , clear);
DECLARE_PNL_PROC_ANS(double, 512, , clear);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
#pragma endregion
StgSharp.Native/sn-head/sn_intrinsic.std.h
+33 -32
View File
@@ -8,11 +8,14 @@
#define static_assert _Static_assert
#endif
#define mat_task_local ((_mat_task_local const *)task)
#define STD_MK_PROC_NAME(T_type, T_arch, T_feature, T_op) \
T_type##_mat_proc_##T_op##_##T_arch##_##T_feature
#define STD_MK_PROC_DECL(T_type, T_arch, T_feature, T_op) \
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL STD_MK_PROC_NAME(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) * task);
T_op)(MAT_TASK(T_type) const *task)
#pragma region param styles
@@ -22,33 +25,32 @@
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_ANS_SCALAR 1
#define _SN_MK_EXPAND_RA(T) \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
#define _SN_MK_EXPAND_RIGHT_ANS(T) \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_LRA(T) \
MAT_KERNEL(T) const *left = (MAT_TASK(T) *)task->mat_0; \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS(T) \
MAT_KERNEL(T) const *left = mat_task_local->mat_0; \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_RAS(T) \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; \
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR(T) \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
void *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_LRAS(T) \
MAT_KERNEL(T) const *left = (MAT_KERNEL(T) *)task->mat_0; \
MAT_KERNEL(T) const *right = (MAT_KERNEL(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR(T) \
MAT_KERNEL(T) const *left = mat_task_local->mat_0; \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
void *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_AS(T) \
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
void *restrict scalar = (scalar_pack const *)task->scalar;
#define _SN_MK_EXPAND_ANS_SCALAR(T) \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
scalar_pack *restrict scalar = &(mat_task_local->padding);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style(T)
#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style##(T)
#pragma endregion
@@ -60,23 +62,22 @@
#define _MK_MKPROC_EXPAND_INDEX_BUFFER \
size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \
size_t count = UNSAFE_AS(size_t, task->padding[1]);
size_t count = UNSAFE_AS(size_t, task->padding.data[1]);
#define _MK_MKPROC_EXPAND_INDEX_KERNEL \
size_t common_i = UNSAFE_AS(size_t, task->common_i); \
size_t common_j = UNSAFE_AS(size_t, task->common_j); \
size_t common_k = UNSAFE_AS(size_t, task->common_k);
#define _SN_MK_EXPAND_MEM(mem) _SN_MK_EXPAND_##mem
#define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem
#define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T
#pragma endregion
#define __mk_param(mem, param, T) \
static_assert((_SN_PARAM_STYLE(param)) - 1 == 0, \
"EXPAND_PARAM: invalid param style '" #param "'"); \
static_assert((_SN_MEM_LAYOUT(mem)) - 1 == 0, \
"EXPAND_PARAM: invalid mem layout '" #mem "'"); \
_SN_MK_EXPAND_PARAM(style, T) \
_SN_MK_EXPAND_MEM(layout)
#define __mk_param_std(mem, param, T) \
static_assert((_SN_PARAM_STYLE(param)) == 1, \
"EXPAND_PARAM: invalid param style '" #param "'"); \
static_assert((_SN_MEM_LAYOUT(mem)) == 1, "EXPAND_PARAM: invalid mem layout '" #mem "'"); \
typedef MAT_TASK(T) _mat_task_local; \
_SN_MK_EXPAND_PARAM(param, T) _SN_MK_EXPAND_MEM(mem)
#endif // !SN_INTRINSIC_PARAM_EXPAND