youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adjust for clang-gcc

Browse Source
This commit is contained in:
Youmu Liu
2026-04-03 18:40:11 +08:00
parent 2f74e5e8a9
commit d702b3224c
19 changed files with 217 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/CMakeLists.txt
+42 -11
View File
@@ -20,9 +20,14 @@ sn_resolve_target_identifiers(
file(GLOB STGSHARP_NATIVE_SOURCES CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-src/*.c)
list(REMOVE_ITEM STGSHARP_NATIVE_SOURCES
${CMAKE_CURRENT_LIST_DIR}/sn-src/formatted.c
${CMAKE_CURRENT_LIST_DIR}/sn-src/StgSharpC.c)
file(GLOB STGSHARP_NATIVE_HEADERS CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-head/*.h)
file(GLOB STGSHARP_NATIVE_TEMPLATES CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.h
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.inl
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.inc
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.ipp)
if(NOT STGSHARP_NATIVE_SOURCES)
message(FATAL_ERROR "No source files found under sn-src/.")
@@ -52,8 +57,14 @@ endif()
set(STGSHARP_BUILD_OUTPUT_DIR ${STGSHARP_SOLUTION_ROOT}/build/${STGSHARP_OS_TAG}-${STGSHARP_ARCH_TAG})
set(STGSHARP_OBJECT_ROOT_DIR ${STGSHARP_SOLUTION_ROOT}/obj/${PROJECT_NAME})
set(STGSHARP_NATIVE_PROJECT_FILES
${STGSHARP_NATIVE_SOURCES}
${STGSHARP_NATIVE_HEADERS}
${STGSHARP_NATIVE_TEMPLATES})
add_library(${PROJECT_NAME} SHARED ${STGSHARP_NATIVE_SOURCES})
add_library(${PROJECT_NAME} SHARED ${STGSHARP_NATIVE_PROJECT_FILES})
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${STGSHARP_NATIVE_PROJECT_FILES})
set_target_properties(${PROJECT_NAME} PROPERTIES
C_STANDARD 11
@@ -75,23 +86,43 @@ endforeach()
set_target_properties(${PROJECT_NAME} PROPERTIES
COMPILE_PDB_OUTPUT_DIRECTORY ${STGSHARP_OBJECT_ROOT_DIR}/$<CONFIG>/${STGSHARP_ARCH_TAG})
if(MSVC OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
if(MSVC OR (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC"))
target_compile_options(${PROJECT_NAME} PRIVATE
/Fd${STGSHARP_OBJECT_ROOT_DIR}/$<CONFIG>/${STGSHARP_ARCH_TAG}/)
endif()
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND SN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
target_compile_options(${PROJECT_NAME} PRIVATE
/clang:-mavx
/clang:-mavx2
/clang:-mfma
/clang:-mavx512f
/clang:-mavx512vl)
# Keep the intrinsic type system and builtin declarations visible to all TUs.
# The runtime dispatch contract still follows T_arch/T_feature at the function level.
if(CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
target_compile_options(${PROJECT_NAME} PRIVATE
/clang:-mavx
/clang:-mavx2
/clang:-mfma
/clang:-mf16c
/clang:-mavx512f
/clang:-mavx512cd
/clang:-mavx512dq
/clang:-mavx512bw
/clang:-mavx512vl)
else()
target_compile_options(${PROJECT_NAME} PRIVATE
-mavx
-mavx2
-mfma
-mf16c
-mavx512f
-mavx512cd
-mavx512dq
-mavx512bw
-mavx512vl)
endif()
endif()
target_include_directories(${PROJECT_NAME}
PRIVATE
${CMAKE_CURRENT_LIST_DIR}/sn-head
${CMAKE_CURRENT_LIST_DIR}/sn-template
${CMAKE_CURRENT_LIST_DIR}/lib)
target_compile_definitions(${PROJECT_NAME}
StgSharp.Native/StgSharp.Native.vcxproj
+1 -1
View File
@@ -110,7 +110,7 @@
<LanguageStandard_C>stdc11</LanguageStandard_C>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;$(ProjectDir)sn-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<!-- DLL 应该用 Windows，不是 Console -->
StgSharp.Native/StgSharp.Native.vcxproj.filters
+4
View File
@@ -16,6 +16,10 @@
<Filter Include="Document">
<UniqueIdentifier>{fbd74b53-5314-4d9d-a788-07bd1b413f74}</UniqueIdentifier>
</Filter>
<Filter Include="模板文件">
<UniqueIdentifier>{C8E0D8C1-42A1-4F7E-9C35-7E6D32D49A7B}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="sn-src\dllmain.c">
StgSharp.Native/sn-head/StgSharpNative.h
+3 -3
View File
@@ -13,7 +13,6 @@
#include "gl.h"
#include "wgl.h"
#define GLFWAPI INTERNAL
#include "glfw\glfw3.h"
#include <immintrin.h>
#include <assert.h>
@@ -71,11 +70,12 @@ SN_API void SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id
SN_API SIMDID SN_DECL sn_get_unite_simd(SIMDID left, SIMDID right);
SN_API int SN_DECL sn_compare_simd(SIMDID left, SIMDID right);
SN_API void SN_DECL sn_set_thread_affinity(int core_id);
SN_API void SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
SN_API void SN_DECL sn_get_numa_each(int numa_id, uint64_t *in_numa_cpuid, uint64_t *global_cpuid);
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *in_numa_cpuid, uint64_t *global_cpuid);
/* Per-core variant: always re-executes CPUID, never caches.
* Must be called on the target core (after affinity is set). */
SN_API SIMDID SN_DECL sn_get_simd_level_local();
SN_API SIMDID SN_DECL sn_get_simd_level_global();
#pragma endregion
#ifdef __cplusplus
StgSharp.Native/sn-head/sn_internal.h
+6 -1
View File
@@ -2,7 +2,7 @@
#ifndef SN_INTERNAL
#define SN_INTERNAL
#include <xmmintrin.h>
#include <immintrin.h>
#define UNSAFE_AS(type, expr) *(type *)&(expr)
@@ -37,4 +37,9 @@
#define PRIVATE static
#define INTERNAL extern
#define SN_PTR_CAST(type, expr) ((type *)(void *)(expr))
#define SN_CONST_PTR_CAST(type, expr) ((type const *)(void const *)(expr))
#define SN_REF_CAST(type, expr) ((type *)(void *)&(expr))
#define SN_CONST_REF_CAST(type, expr) ((type const *)(void const *)&(expr))
#endif //
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+2 -2
View File
@@ -239,8 +239,8 @@ INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma region dot_product
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans);
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(
MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
StgSharp.Native/sn-head/sn_intrinsic.h
+9 -3
View File
@@ -39,6 +39,11 @@ typedef union uint64_u {
uint32_t u32[2];
} uint64_u;
/* Opaque placeholder used by generic single-kernel dispatch.
* It preserves pointer semantics without pretending every branch shares the
* same concrete MAT_KERNEL(T) type. */
typedef struct sn_mat_kernel_opaque sn_mat_kernel_opaque;
typedef struct mat_kernel mat_kernel;
typedef struct mat_panel mat_panel;
@@ -71,9 +76,10 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#endif
typedef void(SN_DECL *UNI_MK_PROC)(matrix_task *task);
typedef void(SN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
mat_kernel *restrict ans, uint64_t scalar);
typedef void(SN_DECL *UNI_MK_PROC)(matrix_task const *task);
typedef void(SN_DECL *UNI_SK_PROC)(sn_mat_kernel_opaque const *left,
sn_mat_kernel_opaque const *right,
sn_mat_kernel_opaque *restrict ans, uint64_t scalar);
typedef void(SN_DECL *VECTORNORMALIZEPROC)(sn_vec_f32 *source, sn_vec_f32 *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
StgSharp.Native/sn-head/sn_intrinsic.matkernel.h
+2 -2
View File
@@ -138,7 +138,7 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
int col_length, int row_length, \
int col_index, int row_index)
/* Declarations for AVX float variant
#if 0
DECLARE_BUILD_PANEL(avx, float);
DECLARE_BUILD_PANEL(512, float);
@@ -149,6 +149,6 @@ DECLARE_STORE_PANEL(512, float);
DECLARE_BUILD_PANEL(512, double);
DECLARE_STORE_PANEL(512, double);
/**/
#endif
#endif /* SN_MATKERNEL */
StgSharp.Native/sn-head/sn_intrinsic.std.h
+20 -9
View File
@@ -11,19 +11,21 @@
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL FORCEINLINE void SN_DECL MK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL MK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(MAT_TASK(T_type) const *task)
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL FORCEINLINE void SN_DECL SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)( \
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op)
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL void SN_DECL SK_PROC_NAME_STD(T_type, T_arch, T_feature, \
T_op)(sn_mat_kernel_opaque const *left_ptr, \
sn_mat_kernel_opaque const *right_ptr, \
sn_mat_kernel_opaque *restrict ans_ptr, \
uint64_t scalar)
#define _SN_STD__PROBE_0 0
#define _SN_STD__PROBE_1 1
@@ -100,4 +102,13 @@
matrix_task const *_ = (_SN_PARAM_STYLE(param), (_SN_MEM_LAYOUT(mem)), task); \
_SN_MK_EXPAND_PARAM(param); \
_SN_MK_EXPAND_MEM(mem)
#define __sk_param_std(T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
SN_MAT_KERNEL_LOCAL const *left = \
(SN_MAT_KERNEL_LOCAL const *)(left_ptr); \
SN_MAT_KERNEL_LOCAL const *right = \
(SN_MAT_KERNEL_LOCAL const *)(right_ptr); \
SN_MAT_KERNEL_LOCAL *restrict ans = \
(SN_MAT_KERNEL_LOCAL *)(ans_ptr)
#endif // !SN_INTRINSIC_PARAM_EXPAND
StgSharp.Native/sn-src/load_intrinsic.x86.c
+5 -6
View File
@@ -18,7 +18,7 @@ PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
SN_API void SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id)
{
sn_null_assert(intrinsic_context);
@@ -74,7 +74,6 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
{
char avx_seg = id.make_byte[5];
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
char is_f16c = (avx_seg & SIMDID_AVX_F16C) != 0;
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
@@ -84,8 +83,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, avx, , scalar_mul);
context->mat_mk[F32].sub = GET_MK_PROC_STD(float, avx, , sub);
context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, avx, , transpose);
context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, , fma) :
GET_MK_PROC_STD(float, avx, _fma, fma);
context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) :
GET_MK_PROC_STD(float, avx, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat_mk[F64].add = GET_MK_PROC_STD(double, avx, , add);
@@ -93,8 +92,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, avx, , scalar_mul);
context->mat_mk[F64].sub = GET_MK_PROC_STD(double, avx, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, , fma) :
GET_MK_PROC_STD(double, avx, _fma, fma);
context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) :
GET_MK_PROC_STD(double, avx, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
StgSharp.Native/sn-src/m.bufop.x86.c
+2 -2
View File
@@ -68,7 +68,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
ans[i].f64_z[1] = vec;
@@ -354,7 +354,7 @@ SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
StgSharp.Native/sn-src/m.transpose.x86.c
+7 -9
View File
@@ -20,10 +20,10 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose)
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
_mm_store_ps(&(ans->f32_x[0]), _mm_movelh_ps(tmp0, tmp2));
_mm_store_ps(&(ans->f32_x[1]), _mm_movehl_ps(tmp2, tmp0));
_mm_store_ps(&(ans->f32_x[2]), _mm_movelh_ps(tmp1, tmp3));
_mm_store_ps(&(ans->f32_x[3]), _mm_movehl_ps(tmp3, tmp1));
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
}
SN_MK_PROC_DECL_STD(float, avx, , transpose)
@@ -38,10 +38,8 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose)
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
_mm256_store_ps(&(ans->f32_y[0]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)));
_mm256_store_ps(&(ans->f32_y[1]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)));
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
SN_MK_PROC_DECL_STD(float, 512, , transpose)
@@ -52,7 +50,7 @@ SN_MK_PROC_DECL_STD(float, 512, , transpose)
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
_mm512_store_ps(&(ans->f32_z[0]), _mm512_permutexvar_ps(transpose_indices, matrix));
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
#endif
StgSharp.Native/sn-src/mk.fma.x86.c
+44 -35
View File
@@ -1,5 +1,5 @@
#include "sn_target.h"
/* clang-format off */
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
@@ -16,7 +16,8 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
@@ -72,7 +73,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
@@ -105,7 +107,8 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
@@ -136,8 +139,11 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps((float *)ans_cur);
register __m512 c_1 = c;
register __m512 c_2 = c;
register __m512 c_3 = c;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
@@ -170,31 +176,31 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
@@ -204,20 +210,23 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1));
register __m128d c0_lo = ans_cur->f64_x[0];
register __m128d c0_hi = ans_cur->f64_x[1];
register __m128d c1_lo = ans_cur->f64_x[2];
register __m128d c1_hi = ans_cur->f64_x[3];
register __m128d c2_lo = ans_cur->f64_x[4];
register __m128d c2_hi = ans_cur->f64_x[5];
register __m128d c3_lo = ans_cur->f64_x[6];
register __m128d c3_hi = ans_cur->f64_x[7];
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const double *right_values = &right_cur->m[0][0];
const double *left_values = &left_cur->m[0][0];
DOUBLE_KERNEL_FMA_CYCLE(0);
@@ -228,21 +237,21 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
ans_cur->f64_x[0] = c0_lo;
ans_cur->f64_x[1] = c0_hi;
ans_cur->f64_x[2] = c1_lo;
ans_cur->f64_x[3] = c1_hi;
ans_cur->f64_x[4] = c2_lo;
ans_cur->f64_x[5] = c2_hi;
ans_cur->f64_x[6] = c3_lo;
ans_cur->f64_x[7] = c3_hi;
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -314,7 +323,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -370,7 +379,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c01 = _mm512_load_pd((double *)&ans_cur->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans_cur->f64_z[1]);
StgSharp.Native/sn-src/s.hash.x86.c
+7 -7
View File
@@ -8,33 +8,33 @@
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
{
uint32_t hash = length, size = length << 1;
byte *end = str + size, *ptr = str;
byte const *end = str + size;
byte const *ptr = str;
if (size < 4) {
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else if (size < 16) {
byte *last = end - 4;
for (; ptr < end; ptr += 4) {
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
}
for (; ptr < end; ptr++) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else {
byte *last_128 = end - 16;
byte *last_32 = end - 4;
byte const *last_128 = end - 16;
byte const *last_32 = end - 4;
uint64_t hash64 = size;
for (; ptr < last_128; ptr += 16) {
__m128i vec = _mm_loadu_si32((int *)ptr);
__m128i vec = _mm_loadu_si128((__m128i const *)ptr);
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
}
hash = _mm_crc32_u8(hash, hash64);
hash = _mm_crc32_u8(hash, hash64 >> 32);
for (; ptr < last_32; ptr += 4) {
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
}
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u8(hash, *ptr);
StgSharp.Native/sn-src/simdid.x86.c
+10
View File
@@ -340,6 +340,16 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#else
SN_API SIMDID SN_DECL sn_get_simd_level_local()
StgSharp.Native/sn-src/sk.fma.x86.c
+41 -30
View File
@@ -14,6 +14,7 @@
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__sk_param_std(float);
register __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
@@ -61,6 +62,7 @@ SN_SK_PROC_DECL_STD(float, sse, , fma)
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
@@ -86,22 +88,23 @@ SN_SK_PROC_DECL_STD(float, avx, , fma)
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps((float *)&left->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps((float *)&left->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps((float *)&left->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps((float *)&left->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
@@ -111,6 +114,7 @@ SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
__sk_param_std(float);
register __m512 c = _mm512_load_ps((float *)ans);
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right);
@@ -137,31 +141,31 @@ SN_SK_PROC_DECL_STD(float, 512, , fma)
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1)); \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left + (1 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left + (2 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left + (3 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
@@ -171,33 +175,38 @@ SN_SK_PROC_DECL_STD(float, 512, , fma)
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
register __m128d c0_lo = _mm_load_pd(ans->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans->f64_x + (3 * 2 + 1));
__sk_param_std(double);
const double *left_values = &left->m[0][0];
const double *right_values = &right->m[0][0];
register __m128d c0_lo = ans->f64_x[0];
register __m128d c0_hi = ans->f64_x[1];
register __m128d c1_lo = ans->f64_x[2];
register __m128d c1_hi = ans->f64_x[3];
register __m128d c2_lo = ans->f64_x[4];
register __m128d c2_hi = ans->f64_x[5];
register __m128d c3_lo = ans->f64_x[6];
register __m128d c3_hi = ans->f64_x[7];
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi);
ans->f64_x[0] = c0_lo;
ans->f64_x[1] = c0_hi;
ans->f64_x[2] = c1_lo;
ans->f64_x[3] = c1_hi;
ans->f64_x[4] = c2_lo;
ans->f64_x[5] = c2_hi;
ans->f64_x[6] = c3_lo;
ans->f64_x[7] = c3_hi;
}
// clang-format off
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
@@ -246,6 +255,7 @@ SN_SK_PROC_DECL_STD(double, avx, , fma)
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
@@ -293,6 +303,7 @@ SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__sk_param_std(double);
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
StgSharp.Native/sn-src/sn_native.c
+1 -1
View File
@@ -44,7 +44,7 @@ SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName)
return (GLFWglproc)ret;
}
SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
SN_API unsigned int SN_DECL linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
{
context->LinkProgram(shaderProgram);
StgSharp.Native/sn-src/thread.linux.c
+9 -2
View File
@@ -29,11 +29,18 @@ SN_API void SN_DECL sn_set_thread_affinity(int core_id)
CPU_FREE(set);
}
SN_API void SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
(void)out_numa_count;
(void)count_of_each_numa;
return 0;
}
SN_API void SN_DECL sn_get_numa_each(int numa_id, int *in_numa_cpuid, int *global_cpuid)
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *in_numa_cpuid, uint64_t *global_cpuid)
{
(void)numa_id;
(void)in_numa_cpuid;
(void)global_cpuid;
return 0;
}
#endif
StgSharp.Native/sn-src/vector.c
+1 -1
View File
@@ -3,7 +3,7 @@
INTERNAL void SN_DECL f32_normalize(sn_vec_f32 *restrict source, sn_vec_f32 *restrict target)
{
register __m128 s = _mm_load_ps(&source->f32_x);
register __m128 s = source->f32_x[0];
register __m128 a_length_sq = _mm_mul_ps(s, s);
a_length_sq = _mm_add_ps(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));