diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj
index b3f14864..d17c7d2a 100644
--- a/StgSharp.Native/StgSharp.Native.vcxproj
+++ b/StgSharp.Native/StgSharp.Native.vcxproj
@@ -162,11 +162,11 @@
-
+
CCode
-
+
@@ -180,6 +180,7 @@
+
diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters
index c282bd74..a5578a5e 100644
--- a/StgSharp.Native/StgSharp.Native.vcxproj.filters
+++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters
@@ -48,7 +48,7 @@
源文件
-
+
源文件
@@ -60,7 +60,7 @@
源文件
-
+
源文件
@@ -101,6 +101,9 @@
头文件
+
+ 头文件
+
diff --git a/StgSharp.Native/sn-head/StgSharpNative.h b/StgSharp.Native/sn-head/StgSharpNative.h
index 7d7d86f2..26fabbf3 100644
--- a/StgSharp.Native/sn-head/StgSharpNative.h
+++ b/StgSharp.Native/sn-head/StgSharpNative.h
@@ -1,4 +1,4 @@
-#ifdef _MSC_VER
+#ifdef _MSC_VER
#pragma once
#endif
@@ -6,6 +6,7 @@
#define SNative
#include "sn_internal.h"
+#include "sn_target.h"
#define STBIDEF SN_API
#include "../lib/stbi/stb_image.h"
@@ -64,7 +65,7 @@ SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName);
SN_API void SN_DECL unloadImageData(Image *out);
SN_API char *SN_DECL readLog(void);
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table);
-SN_API int SN_DECL load_intrinsic_function(void *intrinsic_context);
+SN_API SIMDID SN_DECL load_intrinsic_function(void *intrinsic_context);
#ifdef __cplusplus
}
diff --git a/StgSharp.Native/sn-head/sn_intrincontext.x86.h b/StgSharp.Native/sn-head/sn_intrincontext.x86.h
new file mode 100644
index 00000000..1956b3e3
--- /dev/null
+++ b/StgSharp.Native/sn-head/sn_intrincontext.x86.h
@@ -0,0 +1,189 @@
+#include "sn_target.h"
+
+#if SN_IS_ARCH(SN_ARCH_X86_64)
+
+#include "sn_intrinsic.h"
+
+// -----------------------------------------------------------------------------
+// SIMDID layout helpers (see SIMDID.simd.md)
+// bits [3..0] : Root ISA
+// bits [7..4] : Manufacture (ISA-specific)
+// bits [15..8] : MainLevel (monotonic capability)
+// bits [31..16]: AVX512 Feature
+// bits [39..32]: AMX Feature
+// bits [63..40]: uArch / Policy (optional)
+// -----------------------------------------------------------------------------
+#define SIMDID_SHIFT_ROOT 0
+#define SIMDID_SHIFT_MANU 4
+#define SIMDID_SHIFT_MAIN 8
+#define SIMDID_SHIFT_AVX512 16
+#define SIMDID_SHIFT_AMX 32
+#define SIMDID_SHIFT_UARCH 40
+
+#define SIMDID_MASK_ROOT 0xFULL
+#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
+#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
+#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
+#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
+#define SIMDID_MASK_UARCH (0xFFFFFFULL << SIMDID_SHIFT_UARCH)
+
+// Root ISA (low nibble)
+#define SIMDID_ROOT_UNKNOWN 0x0
+#define SIMDID_ROOT_X86_64 0x1
+#define SIMDID_ROOT_AARCH64 0x2
+#define SIMDID_ROOT_PPC64LE 0x3
+#define SIMDID_ROOT_APPLESILICON 0x4
+#define SIMDID_ROOT_RISCV64 0x5
+#define SIMDID_ROOT_LOONGARCH 0x6
+
+// Manufacture (x86-64 mapping; other ISAs may reinterpret)
+#define SIMDID_MANU_UNKNOWN 0x0
+#define SIMDID_MANU_INTEL 0x1
+#define SIMDID_MANU_AMD 0x2
+#define SIMDID_MANU_VIA 0x3
+
+// MainLevel (ordinal, shared bits 8..15; not bit-flags)
+#define SIMDID_MAIN_LVL_NONE 0x00u
+#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
+#define SIMDID_MAIN_LVL_AVX2 0x02u
+#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u
+#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */
+#define SIMDID_MAIN_LVL_AVX512 0x05u
+#define SIMDID_MAIN_LVL_AMX 0x06u
+#define SIMDID_MAIN_LVL_AVX10 0x07u
+
+#ifndef SIMDID_AVX512_BASE
+#define SIMDID_AVX512_FP16 (1U << 29)
+#define SIMDID_AVX512_BF16 (1U << 28)
+#define SIMDID_AVX512_VNNI (1U << 27)
+#define SIMDID_AVX512_VBMI2 (1U << 26)
+#define SIMDID_AVX512_VBMI (1U << 25)
+#define SIMDID_AVX512_AI_RESERVED (1U << 24)
+#define SIMDID_AVX512_BASE (1U << 20)
+#define SIMDID_AVX512_IMPL_SHIFT 22
+#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
+#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
+#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
+#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
+#endif
+
+#ifndef SIMDID_AMX_TILE
+#define SIMDID_AMX_TILE (1ULL << 32)
+#define SIMDID_AMX_INT8 (1ULL << 33)
+#define SIMDID_AMX_BF16 (1ULL << 34)
+#endif
+
+// Pack helpers (define locally to avoid missing macros when included standalone)
+#ifndef SIMDID_PACK_ROOT
+#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
+#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
+#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
+#define SIMDID_PACK_AVX512(avx512_bits) \
+ (((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
+#define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX)
+#endif
+
+/* Pre-shifted helpers for convenience when constructing SIMDID */
+#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
+#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
+#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
+#define SIMDID_MAIN_ENC_AVX2_FMA SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA)
+#define SIMDID_MAIN_ENC_AVX2_FMA_F16C SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA_F16C)
+#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
+#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
+#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
+
+#pragma region matix function
+
+INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
+
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add);
+
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add);
+
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
+
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
+
+DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
+DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
+DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
+
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
+
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
+
+DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
+
+DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
+
+#pragma endregion
+
+#pragma region fma
+DECLARE_PNL_PROC_ANS(float, sse, , clear);
+DECLARE_PNL_PROC_ANS(float, avx, , clear);
+DECLARE_PNL_PROC_ANS(float, 512, , clear);
+
+DECLARE_PNL_PROC_ANS(double, sse, , clear);
+DECLARE_PNL_PROC_ANS(double, avx, , clear);
+DECLARE_PNL_PROC_ANS(double, 512, , clear);
+
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
+
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
+
+#pragma endregion
+
+#pragma region scaler_simd
+
+INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
+
+#pragma endregion
+
+#pragma region dot_product
+
+INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
+ __m128 const *vector, __m128 *ans);
+INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
+ __m128 *ans);
+INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
+ __m128 *ans);
+INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
+ __m128 *ans);
+
+#pragma endregion
+
+#pragma region string
+
+INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
+INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
+
+#pragma endregion
+
+#else
+
+typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
+
+#endif
diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h
index c02e43ce..2809fed9 100644
--- a/StgSharp.Native/sn-head/sn_intrinsic.h
+++ b/StgSharp.Native/sn-head/sn_intrinsic.h
@@ -94,7 +94,7 @@ typedef struct matrix_task matrix_task;
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__)
- static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
+static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
@@ -108,24 +108,25 @@ typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
-#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
+#define BUF_PROC(T_type, T_arch, T_feature, T_op_name) \
+ T_type##_buf_##T_op_name##_##T_arch##T_feature
-#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
+#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
-#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
+#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
- scalar_pack const *scalar, int count)
-#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
- MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
+ scalar_pack const *scalar, size_t count)
+#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
+ MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
-#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
+#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
-#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
+#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, size_t count)
@@ -141,19 +142,20 @@ typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_pane
scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
-#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
-#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
+#define PNL_PROC(T_type, T_arch, T_feature, T_op_name) \
+ T_type##_pnl_##T_op_name##_##T_arch##T_feature
+#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) *restrict ans)
-#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
+#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
-#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
- INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
- MAT_PANEL_##T_arch##(T_type) const *left, \
- MAT_PANEL_##T_arch##(T_type) const *right, \
+#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
+ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
+ MAT_PANEL_##T_arch##(T_type) const *left, \
+ MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
@@ -288,7 +290,7 @@ typedef enum most_advanced_instruction {
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
-#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
+#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_AVX2 0x02u
#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u
#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */
@@ -310,62 +312,62 @@ typedef enum most_advanced_instruction {
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub);
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul);
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
-DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
-DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill);
-DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
+DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
-DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill);
-DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill);
-DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill);
+DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
+DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
#pragma endregion
#pragma region fma
-DECLARE_PNL_PROC_ANS(float, sse, clear);
-DECLARE_PNL_PROC_ANS(float, avx, clear);
-DECLARE_PNL_PROC_ANS(float, 512, clear);
+DECLARE_PNL_PROC_ANS(float, sse, , clear);
+DECLARE_PNL_PROC_ANS(float, avx, , clear);
+DECLARE_PNL_PROC_ANS(float, 512, , clear);
-DECLARE_PNL_PROC_ANS(double, sse, clear);
-DECLARE_PNL_PROC_ANS(double, avx, clear);
-DECLARE_PNL_PROC_ANS(double, 512, clear);
+DECLARE_PNL_PROC_ANS(double, sse, , clear);
+DECLARE_PNL_PROC_ANS(double, avx, , clear);
+DECLARE_PNL_PROC_ANS(double, 512, , clear);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma);
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
#pragma endregion
diff --git a/StgSharp.Native/sn-src/cpuid.c b/StgSharp.Native/sn-src/cpuid.x86.c
similarity index 88%
rename from StgSharp.Native/sn-src/cpuid.c
rename to StgSharp.Native/sn-src/cpuid.x86.c
index 947d66f3..3a29009c 100644
--- a/StgSharp.Native/sn-src/cpuid.c
+++ b/StgSharp.Native/sn-src/cpuid.x86.c
@@ -1,28 +1,11 @@
#include "sn_target.h"
#include "sn_intrinsic.h"
+static SIMDID global_simdid = { 0 };
+
#if SN_IS_ARCH(SN_ARCH_X86_64)
-#ifndef SIMDID_AVX512_BASE
-#define SIMDID_AVX512_FP16 (1U << 29)
-#define SIMDID_AVX512_BF16 (1U << 28)
-#define SIMDID_AVX512_VNNI (1U << 27)
-#define SIMDID_AVX512_VBMI2 (1U << 26)
-#define SIMDID_AVX512_VBMI (1U << 25)
-#define SIMDID_AVX512_AI_RESERVED (1U << 24)
-#define SIMDID_AVX512_BASE (1U << 20)
-#define SIMDID_AVX512_IMPL_SHIFT 22
-#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
-#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
-#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
-#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
-#endif
-
-#ifndef SIMDID_AMX_TILE
-#define SIMDID_AMX_TILE (1ULL << 32)
-#define SIMDID_AMX_INT8 (1ULL << 33)
-#define SIMDID_AMX_BF16 (1ULL << 34)
-#endif
+#include "sn_intrincontext.x86.h"
#include
#include
@@ -95,8 +78,9 @@ static int is_intel_native_avx512(int family, int model)
SN_API SIMDID SN_DECL sn_get_simd_level()
{
- SIMDID simd_id;
- simd_id.mask = 0;
+ if (!global_simdid.mask) {
+ return global_simdid;
+ }
int eax = 0, ebx = 0, ecx = 0, edx = 0;
@@ -209,9 +193,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
}
}
- simd_id.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
- SIMDID_PACK_AMX(amx_bits);
- return simd_id;
+ global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
+ SIMDID_PACK_AMX(amx_bits);
+ return global_simdid;
}
#else
diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c
deleted file mode 100644
index bd0b5e20..00000000
--- a/StgSharp.Native/sn-src/load_intrinsic.c
+++ /dev/null
@@ -1,155 +0,0 @@
-#include "StgSharpNative.h"
-#include "sn_intrinsic.h"
-
-#if defined(_MSC_VER) && !defined(__clang__)
-#include
-#include
-#else
-#include
-#endif // _MSC_VER
-
-PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
-PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
-PRIVATE void load_intrinsic_512(sn_intrinsic *context);
-
-
-PRIVATE int check_isa(most_advanced_instruction instruction_family)
-{
- if (instruction_family != NEON) {
-#if defined(_MSC_VER) && !defined(__clang__)
- return __check_isa_support(instruction_family, 0);
-#else
- switch (instruction_family) {
- case SSE:
- return __builtin_cpu_supports("sse4.2");
- case AVX256:
- return __builtin_cpu_supports("avx2");
- case AVX512:
- return __builtin_cpu_supports("avx512f");
- default:
- return 0;
- }
-#endif // _MSC_VER
- }
-
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
- return 1;
-#else
- return 0;
-#endif
-}
-
-SN_API int load_intrinsic_function(void *intrinsic_context)
-{
- sn_null_assert(intrinsic_context);
-
- sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
-
- if (check_isa(AVX512)) {
- load_intrinsic_512(context);
- return 3;
- } else if (check_isa(AVX256)) {
- load_intrinsic_avx(context);
- return 2;
- } else if (check_isa(SSE)) {
- load_intrinsic_sse(context);
- return 1;
- } else if (check_isa(NEON)) {
- return -1;
- }
-
- return 0;
-}
-
-void load_intrinsic_sse(sn_intrinsic *context)
-{
- context->city_hash_simplify = city_hash_simplify_sse;
- context->factorial_simd = factorial_simd_sse;
- context->index_pair = index_pair_sse;
-
- context->mat[F32].buffer_add = BUF_PROC(float, sse, add);
- context->mat[F32].buffer_fill = BUF_PROC(float, sse, fill);
- context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, scalar_mul);
- context->mat[F32].buffer_sub = BUF_PROC(float, sse, sub);
- context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose);
- context->mat[F32].build_panel = NULL;
- context->mat[F32].clear_panel = PNL_PROC(float, sse, clear);
- context->mat[F32].panel_fma = PNL_PROC(float, sse, fma);
- context->mat[F32].pivot = pivot_float;
- context->mat[F32].store_panel = NULL;
-
- context->mat[F64].buffer_add = BUF_PROC(double, sse, add);
- context->mat[F64].buffer_fill = BUF_PROC(double, sse, fill);
- context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, scalar_mul);
- context->mat[F64].buffer_sub = BUF_PROC(double, sse, sub);
- context->mat[F64].buffer_transpose = NULL;
- context->mat[F64].build_panel = NULL;
- context->mat[F64].clear_panel = PNL_PROC(double, sse, clear);
- context->mat[F64].panel_fma = PNL_PROC(double, sse, fma);
- context->mat[F64].pivot = NULL;
- context->mat[F64].store_panel = NULL;
-
- context->vec_f32_normalize_3 = f32_normalize;
-}
-
-void load_intrinsic_avx(sn_intrinsic *context)
-{
- context->city_hash_simplify = city_hash_simplify_sse;
- context->factorial_simd = factorial_simd_sse;
- context->index_pair = index_pair_sse;
-
- context->mat[F32].buffer_add = BUF_PROC(float, avx, add);
- context->mat[F32].buffer_fill = BUF_PROC(float, avx, fill);
- context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, scalar_mul);
- context->mat[F32].buffer_sub = BUF_PROC(float, avx, sub);
- context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose);
- context->mat[F32].build_panel = BUILD_PANEL(avx, float);
- context->mat[F32].clear_panel = PNL_PROC(float, avx, clear);
- context->mat[F32].panel_fma = PNL_PROC(float, avx, fma);
- context->mat[F32].pivot = pivot_float;
- context->mat[F32].store_panel = STORE_PANEL(avx, float);
-
- context->mat[F64].buffer_add = BUF_PROC(double, avx, add);
- context->mat[F64].buffer_fill = BUF_PROC(double, avx, fill);
- context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, scalar_mul);
- context->mat[F64].buffer_sub = BUF_PROC(double, avx, sub);
- context->mat[F64].buffer_transpose = NULL;
- context->mat[F64].build_panel = NULL;
- context->mat[F64].clear_panel = PNL_PROC(double, avx, clear);
- context->mat[F64].panel_fma = PNL_PROC(double, avx, fma);
- context->mat[F64].pivot = NULL;
- context->mat[F64].store_panel = NULL;
-
- context->vec_f32_normalize_3 = f32_normalize;
-}
-
-void load_intrinsic_512(sn_intrinsic *context)
-{
- context->city_hash_simplify = city_hash_simplify_sse;
- context->factorial_simd = factorial_simd_sse;
- context->index_pair = index_pair_sse;
-
- context->mat[F32].buffer_add = BUF_PROC(float, 512, add);
- context->mat[F32].buffer_fill = BUF_PROC(float, 512, fill);
- context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, scalar_mul);
- context->mat[F32].buffer_sub = BUF_PROC(float, 512, sub);
- context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose);
- context->mat[F32].build_panel = BUILD_PANEL(512, float);
- context->mat[F32].clear_panel = PNL_PROC(float, 512, clear);
- context->mat[F32].panel_fma = PNL_PROC(float, 512, fma);
- context->mat[F32].pivot = pivot_float;
- context->mat[F32].store_panel = STORE_PANEL(512, float);
-
- context->mat[F64].buffer_add = BUF_PROC(double, 512, add);
- context->mat[F64].buffer_fill = BUF_PROC(double, 512, fill);
- context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, scalar_mul);
- context->mat[F64].buffer_sub = BUF_PROC(double, 512, sub);
- context->mat[F64].buffer_transpose = NULL;
- context->mat[F64].build_panel = BUILD_PANEL(512, double);
- context->mat[F64].clear_panel = PNL_PROC(double, 512, clear);
- context->mat[F64].panel_fma = PNL_PROC(double, 512, fma);
- context->mat[F64].pivot = NULL;
- context->mat[F64].store_panel = STORE_PANEL(512, double);
-
- context->vec_f32_normalize_3 = f32_normalize;
-}
diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c
new file mode 100644
index 00000000..d4e37caa
--- /dev/null
+++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c
@@ -0,0 +1,133 @@
+#include "sn_target.h"
+
+#if SN_IS_ARCH(SN_ARCH_X86_64)
+
+#include "StgSharpNative.h"
+#include "sn_intrinsic.h"
+#include "sn_intrincontext.x86.h"
+
+#if defined(_MSC_VER) && !defined(__clang__)
+#include
+#include
+#else
+#include
+#endif // _MSC_VER
+
+PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
+PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
+PRIVATE void load_intrinsic_avx_fma(sn_intrinsic *context);
+PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID simdid);
+
+SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
+{
+ sn_null_assert(intrinsic_context);
+
+ SIMDID simdid = sn_get_simd_level();
+
+ sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
+
+ if (check_isa(AVX512)) {
+ load_intrinsic_512(context, simdid);
+ } else if (check_isa(AVX256)) {
+ load_intrinsic_avx(context);
+ } else if (check_isa(SSE)) {
+ load_intrinsic_sse(context);
+ }
+
+ return simdid;
+}
+
+void load_intrinsic_sse(sn_intrinsic *context)
+{
+ context->city_hash_simplify = city_hash_simplify_sse;
+ context->factorial_simd = factorial_simd_sse;
+ context->index_pair = index_pair_sse;
+
+ context->mat[F32].buffer_add = BUF_PROC(float, sse, , add);
+ context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
+ context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
+ context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
+ context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose);
+ context->mat[F32].build_panel = NULL;
+ context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
+ context->mat[F32].panel_fma = PNL_PROC(float, sse, , fma);
+ context->mat[F32].pivot = pivot_float;
+ context->mat[F32].store_panel = NULL;
+
+ context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
+ context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
+ context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
+ context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
+ context->mat[F64].buffer_transpose = NULL;
+ context->mat[F64].build_panel = NULL;
+ context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
+ context->mat[F64].panel_fma = PNL_PROC(double, sse, , fma);
+ context->mat[F64].pivot = NULL;
+ context->mat[F64].store_panel = NULL;
+
+ context->vec_f32_normalize_3 = f32_normalize;
+}
+
+void load_intrinsic_avx(sn_intrinsic *context)
+{
+ context->city_hash_simplify = city_hash_simplify_sse;
+ context->factorial_simd = factorial_simd_sse;
+ context->index_pair = index_pair_sse;
+
+ context->mat[F32].buffer_add = BUF_PROC(float, avx, , add);
+ context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
+ context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
+ context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
+ context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose);
+ context->mat[F32].build_panel = BUILD_PANEL(avx, float);
+ context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
+ context->mat[F32].panel_fma = PNL_PROC(float, avx, , fma);
+ context->mat[F32].pivot = pivot_float;
+ context->mat[F32].store_panel = STORE_PANEL(avx, float);
+
+ context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
+ context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
+ context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
+ context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
+ context->mat[F64].buffer_transpose = NULL;
+ context->mat[F64].build_panel = NULL;
+ context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
+ context->mat[F64].panel_fma = PNL_PROC(double, avx, , fma);
+ context->mat[F64].pivot = NULL;
+ context->mat[F64].store_panel = NULL;
+
+ context->vec_f32_normalize_3 = f32_normalize;
+}
+
+void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
+{
+ context->city_hash_simplify = city_hash_simplify_sse;
+ context->factorial_simd = factorial_simd_sse;
+ context->index_pair = index_pair_sse;
+
+ context->mat[F32].buffer_add = BUF_PROC(float, 512, , add);
+ context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
+ context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
+ context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
+ context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose);
+ context->mat[F32].build_panel = BUILD_PANEL(512, float);
+ context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
+ context->mat[F32].panel_fma = PNL_PROC(float, 512, , fma);
+ context->mat[F32].pivot = pivot_float;
+ context->mat[F32].store_panel = STORE_PANEL(512, float);
+
+ context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
+ context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
+ context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
+ context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
+ context->mat[F64].buffer_transpose = NULL;
+ context->mat[F64].build_panel = BUILD_PANEL(512, double);
+ context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
+ context->mat[F64].panel_fma = PNL_PROC(double, 512, , fma);
+ context->mat[F64].pivot = NULL;
+ context->mat[F64].store_panel = STORE_PANEL(512, double);
+
+ context->vec_f32_normalize_3 = f32_normalize;
+}
+
+#endif
diff --git a/StgSharp.Native/sn-src/m.basic.x86.c b/StgSharp.Native/sn-src/m.basic.x86.c
index ea76418f..7d44e6e7 100644
--- a/StgSharp.Native/sn-src/m.basic.x86.c
+++ b/StgSharp.Native/sn-src/m.basic.x86.c
@@ -3,8 +3,9 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
+#include "sn_intrincontext.x86.h"
-DECLARE_PNL_PROC_ANS(float, sse, clear)
+DECLARE_PNL_PROC_ANS(float, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
for (size_t i = 0; i < 4; i++) {
@@ -12,7 +13,7 @@ DECLARE_PNL_PROC_ANS(float, sse, clear)
}
}
-DECLARE_PNL_PROC_ANS(float, avx, clear)
+DECLARE_PNL_PROC_ANS(float, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
@@ -20,7 +21,7 @@ DECLARE_PNL_PROC_ANS(float, avx, clear)
}
}
-DECLARE_PNL_PROC_ANS(float, 512, clear)
+DECLARE_PNL_PROC_ANS(float, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
@@ -28,7 +29,7 @@ DECLARE_PNL_PROC_ANS(float, 512, clear)
}
}
-DECLARE_PNL_PROC_ANS(double, sse, clear)
+DECLARE_PNL_PROC_ANS(double, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
@@ -37,7 +38,7 @@ DECLARE_PNL_PROC_ANS(double, sse, clear)
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
-DECLARE_PNL_PROC_ANS(double, avx, clear)
+DECLARE_PNL_PROC_ANS(double, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
@@ -45,7 +46,7 @@ DECLARE_PNL_PROC_ANS(double, avx, clear)
}
}
-DECLARE_PNL_PROC_ANS(double, 512, clear)
+DECLARE_PNL_PROC_ANS(double, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
diff --git a/StgSharp.Native/sn-src/m.bufop.x86.c b/StgSharp.Native/sn-src/m.bufop.x86.c
index d143d57f..b598f7ad 100644
--- a/StgSharp.Native/sn-src/m.bufop.x86.c
+++ b/StgSharp.Native/sn-src/m.bufop.x86.c
@@ -3,8 +3,9 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
+#include "sn_intrincontext.x86.h"
-DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -15,7 +16,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
}
}
-DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -24,7 +25,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
}
}
-DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -32,7 +33,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
}
}
-DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -47,7 +48,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill)
}
}
-DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -58,7 +59,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill)
}
}
-DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill)
+DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
@@ -67,7 +68,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -82,7 +83,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -93,14 +94,14 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
@@ -131,7 +132,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -146,7 +147,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
@@ -156,7 +157,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -171,7 +172,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -181,7 +182,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -189,7 +190,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
@@ -212,7 +213,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -222,7 +223,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub)
}
}
-DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub)
+DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -230,7 +231,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -246,7 +247,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -258,7 +259,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -268,7 +269,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -292,7 +293,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -308,7 +309,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul)
}
}
-DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, scalar_mul)
+DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, ,scalar_mul)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
for (size_t i = 0; i < count; i++) {
diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c
index e38c90a8..1fe8db8b 100644
--- a/StgSharp.Native/sn-src/m.fma.x86.c
+++ b/StgSharp.Native/sn-src/m.fma.x86.c
@@ -3,12 +3,13 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
+#include "sn_intrincontext.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
{
const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
@@ -58,7 +59,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
// ans = left * right + ans
// right is not transposed
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
@@ -101,7 +102,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma)
// ans = left * right + ans
// right is not transposed
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
@@ -142,7 +143,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
}
}
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
const __m512i ones = _mm512_set1_epi32(1);
@@ -208,7 +209,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
}
}
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
@@ -260,7 +261,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma)
}
}
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
{
register __m256d a0 = _mm256_load_pd((double *)&left->vec[0][0]);
register __m256d a1 = _mm256_load_pd((double *)&left->vec[1][0]);
@@ -301,7 +302,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma)
_mm256_store_pd((double *)&ans->vec[3][0], c3);
}
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
{
register __m256d a0 = _mm256_load_pd((double *)&left->vec[0][0]);
register __m256d a1 = _mm256_load_pd((double *)&left->vec[1][0]);
@@ -344,7 +345,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma)
// ans = left * right + ans
// right is not transposed
-DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, fma)
+DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
{
const __m512i ones = _mm512_set1_epi32(1);