youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adjust for clang-gcc

Browse Source
This commit is contained in:
Youmu Liu
2026-04-03 18:40:11 +08:00
parent 2f74e5e8a9
commit d702b3224c
19 changed files with 217 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Native/sn-src/load_intrinsic.x86.c
+5 -6
View File
@@ -18,7 +18,7 @@ PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void load_intrinsic_function(void *intrinsic_context, uint64_t id)
SN_API void SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id)
{
sn_null_assert(intrinsic_context);
@@ -74,7 +74,6 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
{
char avx_seg = id.make_byte[5];
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
char is_f16c = (avx_seg & SIMDID_AVX_F16C) != 0;
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
@@ -84,8 +83,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, avx, , scalar_mul);
context->mat_mk[F32].sub = GET_MK_PROC_STD(float, avx, , sub);
context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, avx, , transpose);
context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, , fma) :
GET_MK_PROC_STD(float, avx, _fma, fma);
context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) :
GET_MK_PROC_STD(float, avx, , fma);
context->mat_mk[F32].pivot = NULL;
context->mat_mk[F64].add = GET_MK_PROC_STD(double, avx, , add);
@@ -93,8 +92,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, avx, , scalar_mul);
context->mat_mk[F64].sub = GET_MK_PROC_STD(double, avx, , sub);
context->mat_mk[F64].transpose = NULL;
context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, , fma) :
GET_MK_PROC_STD(double, avx, _fma, fma);
context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) :
GET_MK_PROC_STD(double, avx, , fma);
context->mat_mk[F64].pivot = NULL;
context->vec_f32_normalize_3 = f32_normalize;
StgSharp.Native/sn-src/m.bufop.x86.c
+2 -2
View File
@@ -68,7 +68,7 @@ SN_MK_PROC_DECL_STD(double, avx, , fill)
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(BUFFER, ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
ans[i].f64_z[0] = vec;
ans[i].f64_z[1] = vec;
@@ -354,7 +354,7 @@ SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std(BUFFER, RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
register __m512d vec = _mm512_set1_pd(SN_CONST_PTR_CAST(double, scalar->data)[0]);
for (size_t i = 0; i < count; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].f64_z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].f64_z[1], vec);
StgSharp.Native/sn-src/m.transpose.x86.c
+7 -9
View File
@@ -20,10 +20,10 @@ SN_MK_PROC_DECL_STD(float, sse, , transpose)
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
_mm_store_ps(&(ans->f32_x[0]), _mm_movelh_ps(tmp0, tmp2));
_mm_store_ps(&(ans->f32_x[1]), _mm_movehl_ps(tmp2, tmp0));
_mm_store_ps(&(ans->f32_x[2]), _mm_movelh_ps(tmp1, tmp3));
_mm_store_ps(&(ans->f32_x[3]), _mm_movehl_ps(tmp3, tmp1));
ans->f32_x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->f32_x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->f32_x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
}
SN_MK_PROC_DECL_STD(float, avx, , transpose)
@@ -38,10 +38,8 @@ SN_MK_PROC_DECL_STD(float, avx, , transpose)
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
_mm256_store_ps(&(ans->f32_y[0]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0)));
_mm256_store_ps(&(ans->f32_y[1]),
_mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1)));
ans->f32_y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
SN_MK_PROC_DECL_STD(float, 512, , transpose)
@@ -52,7 +50,7 @@ SN_MK_PROC_DECL_STD(float, 512, , transpose)
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
_mm512_store_ps(&(ans->f32_z[0]), _mm512_permutexvar_ps(transpose_indices, matrix));
ans->f32_z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
#endif
StgSharp.Native/sn-src/mk.fma.x86.c
+44 -35
View File
@@ -1,5 +1,5 @@
#include "sn_target.h"
/* clang-format off */
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
@@ -16,7 +16,8 @@ SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]);
@@ -72,7 +73,8 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
@@ -105,7 +107,8 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]);
@@ -136,8 +139,11 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512 c = _mm512_load_ps((float *)ans_cur);
register __m512 c_1 = c;
register __m512 c_2 = c;
register __m512 c_3 = c;
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
@@ -170,31 +176,31 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_cur + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_cur + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_cur + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_cur + (0 * 2 + 1)); \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_cur + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (1 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_cur + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (2 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_cur + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_cur + (3 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
@@ -204,20 +210,23 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur =
ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1));
register __m128d c0_lo = ans_cur->f64_x[0];
register __m128d c0_hi = ans_cur->f64_x[1];
register __m128d c1_lo = ans_cur->f64_x[2];
register __m128d c1_hi = ans_cur->f64_x[3];
register __m128d c2_lo = ans_cur->f64_x[4];
register __m128d c2_hi = ans_cur->f64_x[5];
register __m128d c3_lo = ans_cur->f64_x[6];
register __m128d c3_hi = ans_cur->f64_x[7];
for (size_t z = 0; z < common_z; z++) {
SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z);
SN_MAT_KERNEL_LOCAL const *left_cur = left + GET_INDEX(common_y, z, y);
const double *right_values = &right_cur->m[0][0];
const double *left_values = &left_cur->m[0][0];
DOUBLE_KERNEL_FMA_CYCLE(0);
@@ -228,21 +237,21 @@ SN_MK_PROC_DECL_STD(double, sse, , fma)
DOUBLE_KERNEL_FMA_CYCLE(3);
}
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi);
ans_cur->f64_x[0] = c0_lo;
ans_cur->f64_x[1] = c0_hi;
ans_cur->f64_x[2] = c1_lo;
ans_cur->f64_x[3] = c1_hi;
ans_cur->f64_x[4] = c2_lo;
ans_cur->f64_x[5] = c2_hi;
ans_cur->f64_x[6] = c3_lo;
ans_cur->f64_x[7] = c3_hi;
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // tKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -314,7 +323,7 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]);
@@ -370,7 +379,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(TILE, LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + GET_INDEX(common_y, x, y);
register __m512d c01 = _mm512_load_pd((double *)&ans_cur->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans_cur->f64_z[1]);
StgSharp.Native/sn-src/s.hash.x86.c
+7 -7
View File
@@ -8,33 +8,33 @@
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
{
uint32_t hash = length, size = length << 1;
byte *end = str + size, *ptr = str;
byte const *end = str + size;
byte const *ptr = str;
if (size < 4) {
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else if (size < 16) {
byte *last = end - 4;
for (; ptr < end; ptr += 4) {
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
}
for (; ptr < end; ptr++) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else {
byte *last_128 = end - 16;
byte *last_32 = end - 4;
byte const *last_128 = end - 16;
byte const *last_32 = end - 4;
uint64_t hash64 = size;
for (; ptr < last_128; ptr += 16) {
__m128i vec = _mm_loadu_si32((int *)ptr);
__m128i vec = _mm_loadu_si128((__m128i const *)ptr);
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
}
hash = _mm_crc32_u8(hash, hash64);
hash = _mm_crc32_u8(hash, hash64 >> 32);
for (; ptr < last_32; ptr += 4) {
hash = _mm_crc32_u32(hash, *(uint32_t *)ptr);
hash = _mm_crc32_u32(hash, SN_CONST_PTR_CAST(uint32_t, ptr)[0]);
}
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u8(hash, *ptr);
StgSharp.Native/sn-src/simdid.x86.c
+10
View File
@@ -340,6 +340,16 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#else
SN_API SIMDID SN_DECL sn_get_simd_level_local()
StgSharp.Native/sn-src/sk.fma.x86.c
+41 -30
View File
@@ -14,6 +14,7 @@
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__sk_param_std(float);
register __m128 c0 = _mm_load_ps((float *)&ans->f32_x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->f32_x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->f32_x[2]);
@@ -61,6 +62,7 @@ SN_SK_PROC_DECL_STD(float, sse, , fma)
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
@@ -86,22 +88,23 @@ SN_SK_PROC_DECL_STD(float, avx, , fma)
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]);
register __m256 a0 = _mm256_broadcast_ps((float *)&left->f32_x[0]);
register __m256 a0 = _mm256_broadcast_ps(&left->f32_x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps((float *)&left->f32_x[1]);
register __m256 a1 = _mm256_broadcast_ps(&left->f32_x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps((float *)&left->f32_x[2]);
register __m256 a2 = _mm256_broadcast_ps(&left->f32_x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps((float *)&left->f32_x[3]);
register __m256 a3 = _mm256_broadcast_ps(&left->f32_x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
@@ -111,6 +114,7 @@ SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
__sk_param_std(float);
register __m512 c = _mm512_load_ps((float *)ans);
// full right load：[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right);
@@ -137,31 +141,31 @@ SN_SK_PROC_DECL_STD(float, 512, , fma)
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left + (0 * 2 + 1)); \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left + (1 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left + (2 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left + (3 * 2 + 1)); \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
@@ -171,33 +175,38 @@ SN_SK_PROC_DECL_STD(float, 512, , fma)
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
register __m128d c0_lo = _mm_load_pd(ans->f64_x + (0 * 2 + 0));
register __m128d c0_hi = _mm_load_pd(ans->f64_x + (0 * 2 + 1));
register __m128d c1_lo = _mm_load_pd(ans->f64_x + (1 * 2 + 0));
register __m128d c1_hi = _mm_load_pd(ans->f64_x + (1 * 2 + 1));
register __m128d c2_lo = _mm_load_pd(ans->f64_x + (2 * 2 + 0));
register __m128d c2_hi = _mm_load_pd(ans->f64_x + (2 * 2 + 1));
register __m128d c3_lo = _mm_load_pd(ans->f64_x + (3 * 2 + 0));
register __m128d c3_hi = _mm_load_pd(ans->f64_x + (3 * 2 + 1));
__sk_param_std(double);
const double *left_values = &left->m[0][0];
const double *right_values = &right->m[0][0];
register __m128d c0_lo = ans->f64_x[0];
register __m128d c0_hi = ans->f64_x[1];
register __m128d c1_lo = ans->f64_x[2];
register __m128d c1_hi = ans->f64_x[3];
register __m128d c2_lo = ans->f64_x[4];
register __m128d c2_hi = ans->f64_x[5];
register __m128d c3_lo = ans->f64_x[6];
register __m128d c3_hi = ans->f64_x[7];
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
_mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo);
_mm_store_pd(ans->f64_x + (0 * 2 + 1), c0_hi);
_mm_store_pd(ans->f64_x + (1 * 2 + 0), c1_lo);
_mm_store_pd(ans->f64_x + (1 * 2 + 1), c1_hi);
_mm_store_pd(ans->f64_x + (2 * 2 + 0), c2_lo);
_mm_store_pd(ans->f64_x + (2 * 2 + 1), c2_hi);
_mm_store_pd(ans->f64_x + (3 * 2 + 0), c3_lo);
_mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi);
ans->f64_x[0] = c0_lo;
ans->f64_x[1] = c0_hi;
ans->f64_x[2] = c1_lo;
ans->f64_x[3] = c1_hi;
ans->f64_x[4] = c2_lo;
ans->f64_x[5] = c2_hi;
ans->f64_x[6] = c3_lo;
ans->f64_x[7] = c3_hi;
}
// clang-format off
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
@@ -246,6 +255,7 @@ SN_SK_PROC_DECL_STD(double, avx, , fma)
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->f64_y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->f64_y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->f64_y[2]);
@@ -293,6 +303,7 @@ SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__sk_param_std(double);
register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]);
StgSharp.Native/sn-src/sn_native.c
+1 -1
View File
@@ -44,7 +44,7 @@ SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName)
return (GLFWglproc)ret;
}
SN_API unsigned int linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
SN_API unsigned int SN_DECL linkShaderProgram(GladGLContext *context, GLuint shaderProgram)
{
context->LinkProgram(shaderProgram);
StgSharp.Native/sn-src/thread.linux.c
+9 -2
View File
@@ -29,11 +29,18 @@ SN_API void SN_DECL sn_set_thread_affinity(int core_id)
CPU_FREE(set);
}
SN_API void SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
(void)out_numa_count;
(void)count_of_each_numa;
return 0;
}
SN_API void SN_DECL sn_get_numa_each(int numa_id, int *in_numa_cpuid, int *global_cpuid)
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *in_numa_cpuid, uint64_t *global_cpuid)
{
(void)numa_id;
(void)in_numa_cpuid;
(void)global_cpuid;
return 0;
}
#endif
StgSharp.Native/sn-src/vector.c
+1 -1
View File
@@ -3,7 +3,7 @@
INTERNAL void SN_DECL f32_normalize(sn_vec_f32 *restrict source, sn_vec_f32 *restrict target)
{
register __m128 s = _mm_load_ps(&source->f32_x);
register __m128 s = source->f32_x[0];
register __m128 a_length_sq = _mm_mul_ps(s, s);
a_length_sq = _mm_add_ps(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));