youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mat proc kernel std tamplate is ok, merging old decl

Browse Source
This commit is contained in:
Nitload
2026-03-02 00:24:15 +08:00
parent e198f22712
commit 67fbbde449
8 changed files with 279 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+70 -77
View File
@@ -3,6 +3,7 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.x86.md)
@@ -57,39 +58,39 @@
#define SIMDID_MAIN_LVL_AVX10 0x05u
#ifndef SIMDID_AVX512_BASE
#define SIMDID_AVX512_FP16 U << 13) relative bit 13 within [31..16] */
#define SIMDID_AVX512_BF16 U << 12)
#define SIMDID_AVX512_VNNI U << 11)
#define SIMDID_AVX512_VBMI2 U << 10)
#define SIMDID_AVX512_VBMI U << 9)
#define SIMDID_AVX512_AI_RESERVED U << 8)
#define SIMDID_AVX512_BASE U << 4)
#define SIMDID_AVX512_IMPL_SHIFT
#define SIMDID_AVX512_IMPL_MASK x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE x2U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_FP16 (1U << 13) /* relative bit 13 within [31..16] */
#define SIMDID_AVX512_BF16 (1U << 12)
#define SIMDID_AVX512_VNNI (1U << 11)
#define SIMDID_AVX512_VBMI2 (1U << 10)
#define SIMDID_AVX512_VBMI (1U << 9)
#define SIMDID_AVX512_AI_RESERVED (1U << 8)
#define SIMDID_AVX512_BASE (1U << 4)
#define SIMDID_AVX512_IMPL_SHIFT 6
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
#endif
#ifndef SIMDID_AVX_BASE
#define SIMDID_AVX U << 0) relative bit 0 within [47..40] */
#define SIMDID_AVX2 U << 1)
#define SIMDID_AVX_FMA U << 2)
#define SIMDID_AVX (1U << 0) /* relative bit 0 within [47..40] */
#define SIMDID_AVX2 (1U << 1)
#define SIMDID_AVX_FMA (1U << 2)
#define SIMDID_AVX_F16C (1U << 3)
#define SIMDID_AVX_DUAL ( \
U << 4) glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
#define SIMDID_AVX_DUAL \
(1U << 4) /* glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
#endif
#ifndef SIMDID_AMX_TILE
#define SIMDID_AMX_TILE (1U << 0) relative bit 0 within [39..32] */
#define SIMDID_AMX_TILE (1U << 0) /* relative bit 0 within [39..32] */
#define SIMDID_AMX_INT8 (1U << 1)
#define SIMDID_AMX_BF16 (1U << 2)
#endif
#ifndef SIMDID_AVX10_BASE
#define SIMDID_AVX10 U << 0) /* relative bit 0 within [55..48] */
#define SIMDID_AVX10_V1 U << 1)
#define SIMDID_AVX10_V2 U << 2)
#define SIMDID_AVX10 (1U << 0) /* relative bit 0 within [55..48] */
#define SIMDID_AVX10_V1 (1U << 1)
#define SIMDID_AVX10_V2 (1U << 2)
#define SIMDID_AVX10_256W (1U << 3)
#define SIMDID_AVX10_512W (1U << 4)
#endif
@@ -102,10 +103,10 @@
* 10 = P-core (performance/big core in a hybrid CPU)
* 11 = unified (confirmed non-hybrid; numerically strongest)
* Higher numeric value == stronger / higher-capability topology. */
#define SIMDID_UARCH_HYBRID_E << 0) / E-core (efficiency/small) in a hybrid topology */
#define SIMDID_UARCH_HYBRID_P << 1) / P-core (performance/big) in a hybrid topology */
#define SIMDID_UARCH_SIMD_DUAL ( \
<< 2) / SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
#define SIMDID_UARCH_HYBRID_E (1U << 0) /* E-core (efficiency/small) in a hybrid topology */
#define SIMDID_UARCH_HYBRID_P (1U << 1) /* P-core (performance/big) in a hybrid topology */
#define SIMDID_UARCH_SIMD_DUAL \
(1U << 2) /* SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
#endif
// Pack helpers (define locally to avoid missing macros when included standalone)
@@ -133,71 +134,63 @@
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
STD_MK_PROC_DECL(float, sse, , add);
STD_MK_PROC_DECL(float, avx, , add);
STD_MK_PROC_DECL(float, 512, , add);
SN_MK_PROC_DECL_STD(float, sse, , add);
SN_MK_PROC_DECL_STD(float, avx, , add);
SN_MK_PROC_DECL_STD(float, 512, , add);
STD_MK_PROC_DECL(double, sse, , add);
STD_MK_PROC_DECL(double, avx, , add);
STD_MK_PROC_DECL(double, 512, , add);
SN_MK_PROC_DECL_STD(double, sse, , add);
SN_MK_PROC_DECL_STD(double, avx, , add);
SN_MK_PROC_DECL_STD(double, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
SN_MK_PROC_DECL_STD(float, sse, , sub);
SN_MK_PROC_DECL_STD(float, avx, , sub);
SN_MK_PROC_DECL_STD(float, 512, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
SN_MK_PROC_DECL_STD(double, sse, , sub);
SN_MK_PROC_DECL_STD(double, avx, , sub);
SN_MK_PROC_DECL_STD(double, 512, , sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose);
SN_MK_PROC_DECL_STD(float, sse, , transpose);
SN_MK_PROC_DECL_STD(float, avx, , transpose);
SN_MK_PROC_DECL_STD(float, 512, , transpose);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
SN_MK_PROC_DECL_STD(float, sse, , fill);
SN_MK_PROC_DECL_STD(float, avx, , fill);
SN_MK_PROC_DECL_STD(float, 512, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
SN_MK_PROC_DECL_STD(double, sse, , fill);
SN_MK_PROC_DECL_STD(double, avx, , fill);
SN_MK_PROC_DECL_STD(double, 512, , fill);
DECLARE_PNL_PROC_ANS(float, sse, , clear);
DECLARE_PNL_PROC_ANS(float, avx, , clear);
DECLARE_PNL_PROC_ANS(float, 512, , clear);
SN_MK_PROC_DECL_STD(float, sse, , quality);
DECLARE_PNL_PROC_ANS(double, sse, , clear);
DECLARE_PNL_PROC_ANS(double, avx, , clear);
DECLARE_PNL_PROC_ANS(double, 512, , clear);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
#pragma endregion
StgSharp.Native/sn-head/sn_intrinsic.std.h
+33 -32
View File
@@ -8,11 +8,14 @@
#define static_assert _Static_assert
#endif
#define mat_task_local ((_mat_task_local const *)task)
#define STD_MK_PROC_NAME(T_type, T_arch, T_feature, T_op) \
T_type##_mat_proc_##T_op##_##T_arch##_##T_feature
#define STD_MK_PROC_DECL(T_type, T_arch, T_feature, T_op) \
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL STD_MK_PROC_NAME(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) * task);
T_op)(MAT_TASK(T_type) const *task)
#pragma region param styles
@@ -22,33 +25,32 @@
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_ANS_SCALAR 1
#define _SN_MK_EXPAND_RA(T) \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
#define _SN_MK_EXPAND_RIGHT_ANS(T) \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_LRA(T) \
MAT_KERNEL(T) const *left = (MAT_TASK(T) *)task->mat_0; \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS(T) \
MAT_KERNEL(T) const *left = mat_task_local->mat_0; \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2;
#define _SN_MK_EXPAND_RAS(T) \
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; \
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR(T) \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
void *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_LRAS(T) \
MAT_KERNEL(T) const *left = (MAT_KERNEL(T) *)task->mat_0; \
MAT_KERNEL(T) const *right = (MAT_KERNEL(T) *)task->mat_1; \
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR(T) \
MAT_KERNEL(T) const *left = mat_task_local->mat_0; \
MAT_KERNEL(T) const *right = mat_task_local->mat_1; \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
void *restrict scalar = mat_task_local->scalar;
#define _SN_MK_EXPAND_AS(T) \
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
void *restrict scalar = (scalar_pack const *)task->scalar;
#define _SN_MK_EXPAND_ANS_SCALAR(T) \
MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \
scalar_pack *restrict scalar = &(mat_task_local->padding);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style(T)
#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style##(T)
#pragma endregion
@@ -60,23 +62,22 @@
#define _MK_MKPROC_EXPAND_INDEX_BUFFER \
size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \
size_t count = UNSAFE_AS(size_t, task->padding[1]);
size_t count = UNSAFE_AS(size_t, task->padding.data[1]);
#define _MK_MKPROC_EXPAND_INDEX_KERNEL \
size_t common_i = UNSAFE_AS(size_t, task->common_i); \
size_t common_j = UNSAFE_AS(size_t, task->common_j); \
size_t common_k = UNSAFE_AS(size_t, task->common_k);
#define _SN_MK_EXPAND_MEM(mem) _SN_MK_EXPAND_##mem
#define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem
#define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T
#pragma endregion
#define __mk_param(mem, param, T) \
static_assert((_SN_PARAM_STYLE(param)) - 1 == 0, \
"EXPAND_PARAM: invalid param style '" #param "'"); \
static_assert((_SN_MEM_LAYOUT(mem)) - 1 == 0, \
"EXPAND_PARAM: invalid mem layout '" #mem "'"); \
_SN_MK_EXPAND_PARAM(style, T) \
_SN_MK_EXPAND_MEM(layout)
#define __mk_param_std(mem, param, T) \
static_assert((_SN_PARAM_STYLE(param)) == 1, \
"EXPAND_PARAM: invalid param style '" #param "'"); \
static_assert((_SN_MEM_LAYOUT(mem)) == 1, "EXPAND_PARAM: invalid mem layout '" #mem "'"); \
typedef MAT_TASK(T) _mat_task_local; \
_SN_MK_EXPAND_PARAM(param, T) _SN_MK_EXPAND_MEM(mem)
#endif // !SN_INTRINSIC_PARAM_EXPAND
StgSharp.Native/sn-src/m.basic.x86.c
+5 -52
View File
@@ -3,59 +3,12 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
STD_MK_PROC_DECL(float, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
for (size_t i = 0; i < 4; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
STD_MK_PROC_DECL(float, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
STD_MK_PROC_DECL(float, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
STD_MK_PROC_DECL(double, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
PANEL_COL_VEC((*ans), 1, 0) = zero;
PANEL_COL_VEC((*ans), 2, 0) = zero;
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
STD_MK_PROC_DECL(double, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
STD_MK_PROC_DECL(double, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
PANEL_COL_VEC((*ans), i, 0) = zero;
}
}
// Matrix transpose implementations
STD_MK_PROC_DECL(float, sse, , transpose)
SN_MK_PROC_DECL_STD(float, sse, , transpose)
{
register __m128 col0 = right->f32_x[0];
register __m128 col1 = right->f32_x[1];
@@ -72,7 +25,7 @@ STD_MK_PROC_DECL(float, sse, , transpose)
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
}
STD_MK_PROC_DECL(float, avx, , transpose)
SN_MK_PROC_DECL_STD(float, avx, , transpose)
{
register __m256 col_low = right->f32_y[0];
register __m256 col_high = right->f32_y[1];
@@ -87,7 +40,7 @@ STD_MK_PROC_DECL(float, avx, , transpose)
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
STD_MK_PROC_DECL(float, 512, , transpose)
SN_MK_PROC_DECL_STD(float, 512, , transpose)
{
register __m512 matrix = right->f32_z[0];
StgSharp.Native/sn-src/m.bufop.x86.c
+26 -25
View File
@@ -6,8 +6,9 @@
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
STD_MK_RPOC_DECL(float, sse, , fill)
SN_MK_PROC_DECL_STD(float, sse, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f32_x[0] = vec;
@@ -17,7 +18,7 @@ STD_MK_RPOC_DECL(float, sse, , fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill)
SN_MK_PROC_DECL_STD(float, avx, , fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -26,7 +27,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill)
SN_MK_PROC_DECL_STD(float, 512, , fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -34,7 +35,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill)
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -49,7 +50,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -60,7 +61,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
@@ -69,9 +70,9 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill)
}
}
STD_MK_PROC_DECL(float, sse, , add)
SN_MK_PROC_DECL_STD(float, sse, , add)
{
__mk_param(BUFFER, LEFT_RIGHT_ANS, float);
__mk_param_std(BUFFER, LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -86,7 +87,7 @@ STD_MK_PROC_DECL(float, sse, , add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add)
SN_MK_PROC_DECL_STD(float, avx, , add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -97,14 +98,14 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add)
SN_MK_PROC_DECL_STD(float, 512, , add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add)
SN_MK_PROC_DECL_STD(double, sse, , add)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
@@ -135,7 +136,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add)
SN_MK_PROC_DECL_STD(double, avx, , add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -150,7 +151,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add)
SN_MK_PROC_DECL_STD(double, 512, , add)
{
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
@@ -160,7 +161,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub)
SN_MK_PROC_DECL_STD(float, sse, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -175,7 +176,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub)
SN_MK_PROC_DECL_STD(float, avx, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -185,7 +186,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub)
SN_MK_PROC_DECL_STD(float, 512, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -193,7 +194,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub)
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
@@ -216,7 +217,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub)
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -226,7 +227,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub)
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -234,7 +235,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -250,7 +251,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -262,7 +263,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul)
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -272,7 +273,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul)
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -296,7 +297,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul)
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -312,7 +313,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul)
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
for (size_t i = 0; i < count; i++) {
StgSharp.Native/sn-src/m.fma.x86.c
+26 -9
View File
@@ -3,14 +3,17 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// float
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->f32_x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->f32_x[2]);
@@ -57,8 +60,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
_mm_store_ps((float *)&ans->f32_x[3], c3);
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -103,8 +108,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m256i b01_shift = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -149,8 +156,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
ans->f32_y[1] = c23;
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b_shift = { .m512i_i32 = {
// shuff order
0, 1, 2, 3, // col0
@@ -185,8 +194,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
// double
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
register __m128d c_lo = _mm_load_pd(ans->f64_x + (col * 2 + 0));
@@ -229,8 +240,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
}
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -273,8 +286,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]);
@@ -317,8 +332,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
_mm256_store_pd((double *)&ans->f64_y[3], ccol);
}
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(KERNEL, LEFT_RIGHT_ANS, float);
static const __m512i b01_shift = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/m.fma_tile.x86.c
+8 -8
View File
@@ -11,7 +11,7 @@
// float
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
@@ -73,7 +73,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
_mm_store_ps((float *)&ans_cur->f32_x[3], c3);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -136,7 +136,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
@@ -199,7 +199,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
_mm256_store_ps((float *)&ans_cur->f32_y[1], c23);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
static const __m512i b_shift_source = { .m512i_i32 = {
// shuff order
@@ -279,7 +279,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
@@ -319,7 +319,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
_mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
@@ -397,7 +397,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
int i = enumeration->m128_i32[0];
int j = enumeration->m128_i32[1];
@@ -459,7 +459,7 @@ DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
_mm256_store_pd((double *)&ans_cur->f64_y[3], c3);
}
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } };
static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } };
StgSharp.Native/sn-src/m.solve.x86.c
+7 -4
View File
@@ -1,13 +1,14 @@
#include "sn_target.h"
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include <math.h>
static const float quality_threshold = 1e-6f;
DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
SN_MK_PROC_DECL_STD(float, sse, , try_plu)
{
/* Aliases */
MAT_KERNEL(float) *upper = left; /* U */
@@ -96,7 +97,7 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
_p = _mm_add_ps(_p, _t); \
_t = _mm_movehl_ps(_t, _p); \
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
} while (0)
#define FIND_MAX_F32_SSE(a, b) \
@@ -111,8 +112,10 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(float, sse, , try_plu)
* layout of scalarpack:
*
*/
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality)
SN_MK_PROC_DECL_STD(float, sse, , quality)
{
__mk_param_std(KERNEL, RIGHT_ANS, float);
static const __m128 tiny_source = { .m128_f32 = { 1e-6f, 1e-6f, 1e-6f, 1e-6f } };
register __m128 tiny = _mm_load_ps(tiny_source.m128_f32);