youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add simd id and cpu id

Browse Source
This commit is contained in:
Youmu Liu
2026-01-05 18:10:24 +08:00
parent d0fe4dde1a
commit 88fa653323
11 changed files with 447 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/StgSharp.Native.vcxproj
+3 -2
View File
@@ -162,11 +162,11 @@
<ClCompile Include="sn-src\s.factorial.x86.c" />
<ClCompile Include="sn-src\m.fma.x86.c" />
<ClCompile Include="sn-src\s.hash.x86.c" />
<ClCompile Include="sn-src\load_intrinsic.c">
<ClCompile Include="sn-src\load_intrinsic.x86.c">
<FileType>CCode</FileType>
</ClCompile>
<ClCompile Include="sn-src\load_glfw.c" />
<ClCompile Include="sn-src\cpuid.c" />
<ClCompile Include="sn-src\cpuid.x86.c" />
<ClCompile Include="sn-src\sn_native.c" />
<ClCompile Include="sn-src\str_string.c" />
<ClCompile Include="sn-src\vector.c" />
@@ -180,6 +180,7 @@
<ClInclude Include="sn-head\glfw_function.h" />
<ClInclude Include="sn-head\khrplatform.h" />
<ClInclude Include="sn-head\sn_internal.h" />
<ClInclude Include="sn-head\sn_intrincontext.x86.h" />
<ClInclude Include="sn-head\sn_intrinsic.h" />
<ClInclude Include="sn-head\sn_matkernel.h" />
<ClInclude Include="sn-head\sn_target.h" />
StgSharp.Native/StgSharp.Native.vcxproj.filters
+5 -2
View File
@@ -48,7 +48,7 @@
<ClCompile Include="sn-src\m.fma.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_intrinsic.c">
<ClCompile Include="sn-src\load_intrinsic.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.bufop.x86.c">
@@ -60,7 +60,7 @@
<ClCompile Include="sn-src\m.pivot.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\cpuid.c">
<ClCompile Include="sn-src\cpuid.x86.c">
<Filter>源文件</Filter>
</ClCompile>
</ItemGroup>
@@ -101,6 +101,9 @@
<ClInclude Include="sn-head\sn_target.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_intrincontext.x86.h">
<Filter>头文件</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Library Include="lib\glfw\glfw3.lib">
StgSharp.Native/sn-head/StgSharpNative.h
+3 -2
View File
@@ -1,4 +1,4 @@
#ifdef _MSC_VER
#ifdef _MSC_VER
#pragma once
#endif
@@ -6,6 +6,7 @@
#define SNative
#include "sn_internal.h"
#include "sn_target.h"
#define STBIDEF SN_API
#include "../lib/stbi/stb_image.h"
@@ -64,7 +65,7 @@ SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName);
SN_API void SN_DECL unloadImageData(Image *out);
SN_API char *SN_DECL readLog(void);
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table);
SN_API int SN_DECL load_intrinsic_function(void *intrinsic_context);
SN_API SIMDID SN_DECL load_intrinsic_function(void *intrinsic_context);
#ifdef __cplusplus
}
StgSharp.Native/sn-head/sn_intrincontext.x86.h
+189
View File
@@ -0,0 +1,189 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.simd.md)
// bits [3..0] : Root ISA
// bits [7..4] : Manufacture (ISA-specific)
// bits [15..8] : MainLevel (monotonic capability)
// bits [31..16]: AVX512 Feature
// bits [39..32]: AMX Feature
// bits [63..40]: uArch / Policy (optional)
// -----------------------------------------------------------------------------
#define SIMDID_SHIFT_ROOT 0
#define SIMDID_SHIFT_MANU 4
#define SIMDID_SHIFT_MAIN 8
#define SIMDID_SHIFT_AVX512 16
#define SIMDID_SHIFT_AMX 32
#define SIMDID_SHIFT_UARCH 40
#define SIMDID_MASK_ROOT 0xFULL
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
#define SIMDID_MASK_UARCH (0xFFFFFFULL << SIMDID_SHIFT_UARCH)
// Root ISA (low nibble)
#define SIMDID_ROOT_UNKNOWN 0x0
#define SIMDID_ROOT_X86_64 0x1
#define SIMDID_ROOT_AARCH64 0x2
#define SIMDID_ROOT_PPC64LE 0x3
#define SIMDID_ROOT_APPLESILICON 0x4
#define SIMDID_ROOT_RISCV64 0x5
#define SIMDID_ROOT_LOONGARCH 0x6
// Manufacture (x86-64 mapping; other ISAs may reinterpret)
#define SIMDID_MANU_UNKNOWN 0x0
#define SIMDID_MANU_INTEL 0x1
#define SIMDID_MANU_AMD 0x2
#define SIMDID_MANU_VIA 0x3
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_AVX2 0x02u
#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u
#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */
#define SIMDID_MAIN_LVL_AVX512 0x05u
#define SIMDID_MAIN_LVL_AMX 0x06u
#define SIMDID_MAIN_LVL_AVX10 0x07u
#ifndef SIMDID_AVX512_BASE
#define SIMDID_AVX512_FP16 (1U << 29)
#define SIMDID_AVX512_BF16 (1U << 28)
#define SIMDID_AVX512_VNNI (1U << 27)
#define SIMDID_AVX512_VBMI2 (1U << 26)
#define SIMDID_AVX512_VBMI (1U << 25)
#define SIMDID_AVX512_AI_RESERVED (1U << 24)
#define SIMDID_AVX512_BASE (1U << 20)
#define SIMDID_AVX512_IMPL_SHIFT 22
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
#endif
#ifndef SIMDID_AMX_TILE
#define SIMDID_AMX_TILE (1ULL << 32)
#define SIMDID_AMX_INT8 (1ULL << 33)
#define SIMDID_AMX_BF16 (1ULL << 34)
#endif
// Pack helpers (define locally to avoid missing macros when included standalone)
#ifndef SIMDID_PACK_ROOT
#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
#define SIMDID_PACK_AVX512(avx512_bits) \
(((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
#define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX)
#endif
/* Pre-shifted helpers for convenience when constructing SIMDID */
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
#define SIMDID_MAIN_ENC_AVX2_FMA SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA)
#define SIMDID_MAIN_ENC_AVX2_FMA_F16C SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2_FMA_F16C)
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
#pragma region matix function
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
#pragma endregion
#pragma region fma
DECLARE_PNL_PROC_ANS(float, sse, , clear);
DECLARE_PNL_PROC_ANS(float, avx, , clear);
DECLARE_PNL_PROC_ANS(float, 512, , clear);
DECLARE_PNL_PROC_ANS(double, sse, , clear);
DECLARE_PNL_PROC_ANS(double, avx, , clear);
DECLARE_PNL_PROC_ANS(double, 512, , clear);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
#pragma endregion
#pragma region scaler_simd
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
#pragma endregion
#pragma region string
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
#pragma endregion
#else
typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
#endif
StgSharp.Native/sn-head/sn_intrinsic.h
+64 -62
View File
@@ -94,7 +94,7 @@ typedef struct matrix_task matrix_task;
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__)
static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
#endif
#pragma region matrix buffer operation function type
@@ -108,24 +108,25 @@ typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
int count);
#define BUF_PROC(T_type, T_arch, T_op_name) T_type##_buf_##T_op_name##_##T_arch
#define BUF_PROC(T_type, T_arch, T_feature, T_op_name) \
T_type##_buf_##T_op_name##_##T_arch##T_feature
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
scalar_pack const *scalar, int count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_op_name)( \
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, size_t count)
@@ -141,19 +142,20 @@ typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_pane
scalar_pack *scalar);
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
#define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch
#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
#define PNL_PROC(T_type, T_arch, T_feature, T_op_name) \
T_type##_pnl_##T_op_name##_##T_arch##T_feature
#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans)
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
MAT_PANEL_##T_arch##(T_type) const *left, \
MAT_PANEL_##T_arch##(T_type) const *right, \
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
#pragma endregion
@@ -288,7 +290,7 @@ typedef enum most_advanced_instruction {
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_AVX2 0x02u
#define SIMDID_MAIN_LVL_AVX2_FMA 0x03u
#define SIMDID_MAIN_LVL_AVX2_FMA_F16C 0x04u /* 16-bit scalar/vector ready */
@@ -310,62 +312,62 @@ typedef enum most_advanced_instruction {
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, avx, transpose);
DECLARE_KER_PROC_RIGHT_ANS(float, 512, transpose);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
#pragma endregion
#pragma region fma
DECLARE_PNL_PROC_ANS(float, sse, clear);
DECLARE_PNL_PROC_ANS(float, avx, clear);
DECLARE_PNL_PROC_ANS(float, 512, clear);
DECLARE_PNL_PROC_ANS(float, sse, , clear);
DECLARE_PNL_PROC_ANS(float, avx, , clear);
DECLARE_PNL_PROC_ANS(float, 512, , clear);
DECLARE_PNL_PROC_ANS(double, sse, clear);
DECLARE_PNL_PROC_ANS(double, avx, clear);
DECLARE_PNL_PROC_ANS(double, 512, clear);
DECLARE_PNL_PROC_ANS(double, sse, , clear);
DECLARE_PNL_PROC_ANS(double, avx, , clear);
DECLARE_PNL_PROC_ANS(double, 512, , clear);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
#pragma endregion
StgSharp.Native/sn-src/cpuid.c → StgSharp.Native/sn-src/cpuid.x86.c
+9 -25
View File
@@ -1,28 +1,11 @@
#include "sn_target.h"
#include "sn_intrinsic.h"
static SIMDID global_simdid = { 0 };
#if SN_IS_ARCH(SN_ARCH_X86_64)
#ifndef SIMDID_AVX512_BASE
#define SIMDID_AVX512_FP16 (1U << 29)
#define SIMDID_AVX512_BF16 (1U << 28)
#define SIMDID_AVX512_VNNI (1U << 27)
#define SIMDID_AVX512_VBMI2 (1U << 26)
#define SIMDID_AVX512_VBMI (1U << 25)
#define SIMDID_AVX512_AI_RESERVED (1U << 24)
#define SIMDID_AVX512_BASE (1U << 20)
#define SIMDID_AVX512_IMPL_SHIFT 22
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
#endif
#ifndef SIMDID_AMX_TILE
#define SIMDID_AMX_TILE (1ULL << 32)
#define SIMDID_AMX_INT8 (1ULL << 33)
#define SIMDID_AMX_BF16 (1ULL << 34)
#endif
#include "sn_intrincontext.x86.h"
#include <immintrin.h>
#include <string.h>
@@ -95,8 +78,9 @@ static int is_intel_native_avx512(int family, int model)
SN_API SIMDID SN_DECL sn_get_simd_level()
{
SIMDID simd_id;
simd_id.mask = 0;
if (!global_simdid.mask) {
return global_simdid;
}
int eax = 0, ebx = 0, ecx = 0, edx = 0;
@@ -209,9 +193,9 @@ SN_API SIMDID SN_DECL sn_get_simd_level()
}
}
simd_id.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
SIMDID_PACK_AMX(amx_bits);
return simd_id;
global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) |
SIMDID_PACK_AMX(amx_bits);
return global_simdid;
}
#else
StgSharp.Native/sn-src/load_intrinsic.c
-155
View File
@@ -1,155 +0,0 @@
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#include <isa_availability.h>
#else
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
PRIVATE void load_intrinsic_512(sn_intrinsic *context);
PRIVATE int check_isa(most_advanced_instruction instruction_family)
{
if (instruction_family != NEON) {
#if defined(_MSC_VER) && !defined(__clang__)
return __check_isa_support(instruction_family, 0);
#else
switch (instruction_family) {
case SSE:
return __builtin_cpu_supports("sse4.2");
case AVX256:
return __builtin_cpu_supports("avx2");
case AVX512:
return __builtin_cpu_supports("avx512f");
default:
return 0;
}
#endif // _MSC_VER
}
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
return 1;
#else
return 0;
#endif
}
SN_API int load_intrinsic_function(void *intrinsic_context)
{
sn_null_assert(intrinsic_context);
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
if (check_isa(AVX512)) {
load_intrinsic_512(context);
return 3;
} else if (check_isa(AVX256)) {
load_intrinsic_avx(context);
return 2;
} else if (check_isa(SSE)) {
load_intrinsic_sse(context);
return 1;
} else if (check_isa(NEON)) {
return -1;
}
return 0;
}
void load_intrinsic_sse(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, sse, add);
context->mat[F32].buffer_fill = BUF_PROC(float, sse, fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, sse, sub);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose);
context->mat[F32].build_panel = NULL;
context->mat[F32].clear_panel = PNL_PROC(float, sse, clear);
context->mat[F32].panel_fma = PNL_PROC(float, sse, fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, sse, add);
context->mat[F64].buffer_fill = BUF_PROC(double, sse, fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, sse, sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, sse, clear);
context->mat[F64].panel_fma = PNL_PROC(double, sse, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, avx, add);
context->mat[F32].buffer_fill = BUF_PROC(float, avx, fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, avx, sub);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose);
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
context->mat[F32].clear_panel = PNL_PROC(float, avx, clear);
context->mat[F32].panel_fma = PNL_PROC(float, avx, fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(avx, float);
context->mat[F64].buffer_add = BUF_PROC(double, avx, add);
context->mat[F64].buffer_fill = BUF_PROC(double, avx, fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, avx, sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, clear);
context->mat[F64].panel_fma = PNL_PROC(double, avx, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_512(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, 512, add);
context->mat[F32].buffer_fill = BUF_PROC(float, 512, fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, 512, sub);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose);
context->mat[F32].build_panel = BUILD_PANEL(512, float);
context->mat[F32].clear_panel = PNL_PROC(float, 512, clear);
context->mat[F32].panel_fma = PNL_PROC(float, 512, fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(512, float);
context->mat[F64].buffer_add = BUF_PROC(double, 512, add);
context->mat[F64].buffer_fill = BUF_PROC(double, 512, fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, 512, sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = BUILD_PANEL(512, double);
context->mat[F64].clear_panel = PNL_PROC(double, 512, clear);
context->mat[F64].panel_fma = PNL_PROC(double, 512, fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = STORE_PANEL(512, double);
context->vec_f32_normalize_3 = f32_normalize;
}
StgSharp.Native/sn-src/load_intrinsic.x86.c
+133
View File
@@ -0,0 +1,133 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#if defined(_MSC_VER) && !defined(__clang__)
#include <intrin.h>
#include <isa_availability.h>
#else
#include <cpuid.h>
#endif // _MSC_VER
PRIVATE void load_intrinsic_sse(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context);
PRIVATE void load_intrinsic_avx_fma(sn_intrinsic *context);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID simdid);
SN_API SIMDID load_intrinsic_function(void *intrinsic_context)
{
sn_null_assert(intrinsic_context);
SIMDID simdid = sn_get_simd_level();
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
if (check_isa(AVX512)) {
load_intrinsic_512(context, simdid);
} else if (check_isa(AVX256)) {
load_intrinsic_avx(context);
} else if (check_isa(SSE)) {
load_intrinsic_sse(context);
}
return simdid;
}
void load_intrinsic_sse(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, sse, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, sse, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, sse, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, sse, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, sse, transpose);
context->mat[F32].build_panel = NULL;
context->mat[F32].clear_panel = PNL_PROC(float, sse, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, sse, , fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = NULL;
context->mat[F64].buffer_add = BUF_PROC(double, sse, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, sse, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, sse, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, sse, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, sse, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, sse, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(sn_intrinsic *context)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, avx, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, avx, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, avx, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, avx, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, avx, transpose);
context->mat[F32].build_panel = BUILD_PANEL(avx, float);
context->mat[F32].clear_panel = PNL_PROC(float, avx, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, avx, , fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(avx, float);
context->mat[F64].buffer_add = BUF_PROC(double, avx, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, avx, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, avx, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, avx, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = NULL;
context->mat[F64].clear_panel = PNL_PROC(double, avx, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, avx, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->index_pair = index_pair_sse;
context->mat[F32].buffer_add = BUF_PROC(float, 512, , add);
context->mat[F32].buffer_fill = BUF_PROC(float, 512, , fill);
context->mat[F32].buffer_scalar_mul = BUF_PROC(float, 512, , scalar_mul);
context->mat[F32].buffer_sub = BUF_PROC(float, 512, , sub);
context->mat[F32].buffer_transpose = KER_PROC(float, 512, transpose);
context->mat[F32].build_panel = BUILD_PANEL(512, float);
context->mat[F32].clear_panel = PNL_PROC(float, 512, , clear);
context->mat[F32].panel_fma = PNL_PROC(float, 512, , fma);
context->mat[F32].pivot = pivot_float;
context->mat[F32].store_panel = STORE_PANEL(512, float);
context->mat[F64].buffer_add = BUF_PROC(double, 512, , add);
context->mat[F64].buffer_fill = BUF_PROC(double, 512, , fill);
context->mat[F64].buffer_scalar_mul = BUF_PROC(double, 512, , scalar_mul);
context->mat[F64].buffer_sub = BUF_PROC(double, 512, , sub);
context->mat[F64].buffer_transpose = NULL;
context->mat[F64].build_panel = BUILD_PANEL(512, double);
context->mat[F64].clear_panel = PNL_PROC(double, 512, , clear);
context->mat[F64].panel_fma = PNL_PROC(double, 512, , fma);
context->mat[F64].pivot = NULL;
context->mat[F64].store_panel = STORE_PANEL(512, double);
context->vec_f32_normalize_3 = f32_normalize;
}
#endif
StgSharp.Native/sn-src/m.basic.x86.c
+7 -6
View File
@@ -3,8 +3,9 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
DECLARE_PNL_PROC_ANS(float, sse, clear)
DECLARE_PNL_PROC_ANS(float, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
for (size_t i = 0; i < 4; i++) {
@@ -12,7 +13,7 @@ DECLARE_PNL_PROC_ANS(float, sse, clear)
}
}
DECLARE_PNL_PROC_ANS(float, avx, clear)
DECLARE_PNL_PROC_ANS(float, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
@@ -20,7 +21,7 @@ DECLARE_PNL_PROC_ANS(float, avx, clear)
}
}
DECLARE_PNL_PROC_ANS(float, 512, clear)
DECLARE_PNL_PROC_ANS(float, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
@@ -28,7 +29,7 @@ DECLARE_PNL_PROC_ANS(float, 512, clear)
}
}
DECLARE_PNL_PROC_ANS(double, sse, clear)
DECLARE_PNL_PROC_ANS(double, sse, , clear)
{
register __m128 zero = _mm_setzero_ps();
PANEL_COL_VEC((*ans), 0, 0) = zero;
@@ -37,7 +38,7 @@ DECLARE_PNL_PROC_ANS(double, sse, clear)
PANEL_COL_VEC((*ans), 3, 0) = zero;
}
DECLARE_PNL_PROC_ANS(double, avx, clear)
DECLARE_PNL_PROC_ANS(double, avx, , clear)
{
register __m256 zero = _mm256_setzero_ps();
for (size_t i = 0; i < 8; i++) {
@@ -45,7 +46,7 @@ DECLARE_PNL_PROC_ANS(double, avx, clear)
}
}
DECLARE_PNL_PROC_ANS(double, 512, clear)
DECLARE_PNL_PROC_ANS(double, 512, , clear)
{
register __m512 zero = _mm512_setzero_ps();
for (size_t i = 0; i < 16; i++) {
StgSharp.Native/sn-src/m.bufop.x86.c
+25 -24
View File
@@ -3,8 +3,9 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -15,7 +16,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, sse, fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -24,7 +25,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, avx, fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -32,7 +33,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill)
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -47,7 +48,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill)
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -58,7 +59,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, avx, fill)
}
}
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill)
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
for (size_t i = 0; i < count; i++) {
@@ -67,7 +68,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, 512, fill)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -82,7 +83,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -93,14 +94,14 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
ans[i].f32_z[0] = c0;
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
@@ -131,7 +132,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
{
for (size_t i = 0; i < count; i++) {
size_t s = sizeof(MAT_KERNEL(float));
@@ -146,7 +147,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
{
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
@@ -156,7 +157,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, add)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
@@ -171,7 +172,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -181,7 +182,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -189,7 +190,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
@@ -212,7 +213,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
@@ -222,7 +223,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, sub)
}
}
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub)
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
{
for (size_t i = 0; i < count; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
@@ -230,7 +231,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, sub)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
{
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -246,7 +247,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
{
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -258,7 +259,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
{
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -268,7 +269,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
{
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
for (size_t i = 0; i < count; i++) {
@@ -292,7 +293,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
{
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
for (size_t i = 0; i < count; i++) {
@@ -308,7 +309,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, scalar_mul)
}
}
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, scalar_mul)
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, ,scalar_mul)
{
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
for (size_t i = 0; i < count; i++) {
StgSharp.Native/sn-src/m.fma.x86.c
+9 -8
View File
@@ -3,12 +3,13 @@
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrincontext.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, , fma)
{
const __m128 a0 = PANEL_COL_VEC((*left), 0, 0);
const __m128 a1 = PANEL_COL_VEC((*left), 1, 0);
@@ -58,7 +59,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma)
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
@@ -101,7 +102,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma)
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, , fma)
{
static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 };
@@ -142,7 +143,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma)
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, , fma)
{
const __m512i ones = _mm512_set1_epi32(1);
@@ -208,7 +209,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma)
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma)
{
for (int col = 0; col < 4; ++col) {
/* current answer column (two chunks: rows 0-1 and 2-3) */
@@ -260,7 +261,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, fma)
}
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, , fma)
{
register __m256d a0 = _mm256_load_pd((double *)&left->vec[0][0]);
register __m256d a1 = _mm256_load_pd((double *)&left->vec[1][0]);
@@ -301,7 +302,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, fma)
_mm256_store_pd((double *)&ans->vec[3][0], c3);
}
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma)
{
register __m256d a0 = _mm256_load_pd((double *)&left->vec[0][0]);
register __m256d a1 = _mm256_load_pd((double *)&left->vec[1][0]);
@@ -344,7 +345,7 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, avx_fma, fma)
// ans = left * right + ans
// right is not transposed
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, fma)
DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, 512, , fma)
{
const __m512i ones = _mm512_set1_epi32(1);