From 2741b3458363e26ae13f50162a7d34e542923d26 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 12 Jun 2026 18:25:04 +0800 Subject: [PATCH] left and right use diff pack kernel --- .../Internal/IntrinsicContext.cs | 4 +- .../sn-head/sn_intrinsic.context.x86.h | 26 +- StgSharp.Native/sn-head/sn_intrinsic.h | 29 +- StgSharp.Native/sn-src/load_intrinsic.x86.c | 84 ++--- StgSharp.Native/sn-src/mk.fma.pack.c | 288 ++++++++++++++++++ StgSharp.Native/sn-src/mk.fma.x86.c | 123 ++------ 6 files changed, 390 insertions(+), 164 deletions(-) create mode 100644 StgSharp.Native/sn-src/mk.fma.pack.c diff --git a/StgSharp.Mathematics/Internal/IntrinsicContext.cs b/StgSharp.Mathematics/Internal/IntrinsicContext.cs index c7b30c15..eaedd2b1 100644 --- a/StgSharp.Mathematics/Internal/IntrinsicContext.cs +++ b/StgSharp.Mathematics/Internal/IntrinsicContext.cs @@ -42,7 +42,9 @@ namespace StgSharp.Mathematics.Internal Transpose = 4, ClearPanel = 5, Fma = 6, - Pivot = 8, + PackFmaLeft = 7, + PackFmaRight = 8, + Pivot = 9, } diff --git a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h index 7a8b6baa..b8883461 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h @@ -175,17 +175,39 @@ SN_MK_PROC_DECL_STD(float, sse, , quality); * * CONVENSIONS: * -* buffer_1 is pointer to original colum, offset_1 is the offset to the original column, count_1 is the count can be packed +* buffer_1 is pointer to original colum, +* offset_1 is the offset to the original column, +* count_1 is the count can be packed +* * C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension, * and make sure that count_1 will not exceeds lenght of the column * * scalar pack carries column length and offset data: -* [column length, offset, buffer requires, empty] +* [column length, empty, empty, empty] */ SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left); SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left); SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left); +/* +* pack right matrix data to L4 buffer +* +* CONVENSIONS: +* +* buffer_1 is pointer to original row, +* offset_1 is the offset to the original row, +* count_1 is the count can be packed +* +* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension, +* and make sure that count_1 will not exceeds lenght of the row +* +* scalar pack carries column length and offset data: +* [column length, empty, empty, empty] +*/ +SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right); +SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right); +SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right); + SN_MK_PROC_DECL_STD(float, sse, , fma); SN_MK_PROC_DECL_STD(float, avx, , fma); SN_MK_PROC_DECL_STD(float, avx, _fma, fma); diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 5abcd85a..307cd5e1 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -160,23 +160,11 @@ typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length #pragma endregion typedef struct mat_mk_intrinsic { - UNI_MK_PROC add; - UNI_MK_PROC fill; - UNI_MK_PROC scalar_mul; - UNI_MK_PROC sub; - UNI_MK_PROC transpose; - UNI_MK_PROC fma; - UNI_MK_PROC pivot; + UNI_MK_PROC proc_array[10]; } mat_mk_intrinsic; typedef struct mat_sk_intrinsic { - UNI_SK_PROC add; - UNI_SK_PROC fill; - UNI_SK_PROC scalar_mul; - UNI_SK_PROC sub; - UNI_SK_PROC transpose; - UNI_SK_PROC fma; - UNI_SK_PROC pivot; + UNI_SK_PROC proc_array[10]; } mat_sk_intrinsic; // Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync. @@ -188,6 +176,19 @@ typedef struct sn_intrinsic { mat_sk_intrinsic mat_sk[0x0a]; } sn_intrinsic; + +typedef enum mat_proc_index { + Add = 0, + Fill = 1, + ScalarMul = 2, + Sub = 3, + Transpose = 4, + ClearPanel = 5, + Fma = 6, + PackFmaLeft = 7, + PackFmaRight = 8, + Pivot = 9, +} mat_proc_index; typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask; #endif diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index ef1a1db5..dd04839f 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -44,21 +44,21 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; - context->mat_mk[F32].add = GET_MK_PROC_STD(float, sse, , add); - context->mat_mk[F32].fill = GET_MK_PROC_STD(float, sse, , fill); - context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, sse, , scalar_mul); - context->mat_mk[F32].sub = GET_MK_PROC_STD(float, sse, , sub); - context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, sse, , transpose); - context->mat_mk[F32].fma = GET_MK_PROC_STD(float, sse, , fma); - context->mat_mk[F32].pivot = NULL; + context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, sse, , add); + context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, sse, , fill); + context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, sse, , scalar_mul); + context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, sse, , sub); + context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, sse, , transpose); + context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, sse, , fma); + context->mat_mk[F32].proc_array[Pivot] = NULL; - context->mat_mk[F64].add = GET_MK_PROC_STD(double, sse, , add); - context->mat_mk[F64].fill = GET_MK_PROC_STD(double, sse, , fill); - context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, sse, , scalar_mul); - context->mat_mk[F64].sub = GET_MK_PROC_STD(double, sse, , sub); - context->mat_mk[F64].transpose = NULL; - context->mat_mk[F64].fma = GET_MK_PROC_STD(double, sse, , fma); - context->mat_mk[F64].pivot = NULL; + context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, sse, , add); + context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, sse, , fill); + context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, sse, , scalar_mul); + context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, sse, , sub); + context->mat_mk[F64].proc_array[Transpose] = NULL; + context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, sse, , fma); + context->mat_mk[F64].proc_array[Pivot] = NULL; context->vec_f32_normalize_3 = f32_normalize; } @@ -71,23 +71,23 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; - context->mat_mk[F32].add = GET_MK_PROC_STD(float, avx, , add); - context->mat_mk[F32].fill = GET_MK_PROC_STD(float, avx, , fill); - context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, avx, , scalar_mul); - context->mat_mk[F32].sub = GET_MK_PROC_STD(float, avx, , sub); - context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, avx, , transpose); - context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) : + context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, avx, , add); + context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, avx, , fill); + context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, avx, , scalar_mul); + context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, avx, , sub); + context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, avx, , transpose); + context->mat_mk[F32].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) : GET_MK_PROC_STD(float, avx, , fma); - context->mat_mk[F32].pivot = NULL; + context->mat_mk[F32].proc_array[Pivot] = NULL; - context->mat_mk[F64].add = GET_MK_PROC_STD(double, avx, , add); - context->mat_mk[F64].fill = GET_MK_PROC_STD(double, avx, , fill); - context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, avx, , scalar_mul); - context->mat_mk[F64].sub = GET_MK_PROC_STD(double, avx, , sub); - context->mat_mk[F64].transpose = NULL; - context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) : + context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, avx, , add); + context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, avx, , fill); + context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, avx, , scalar_mul); + context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, avx, , sub); + context->mat_mk[F64].proc_array[Transpose] = NULL; + context->mat_mk[F64].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) : GET_MK_PROC_STD(double, avx, , fma); - context->mat_mk[F64].pivot = NULL; + context->mat_mk[F64].proc_array[Pivot] = NULL; context->vec_f32_normalize_3 = f32_normalize; } @@ -97,21 +97,21 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; - context->mat_mk[F32].add = GET_MK_PROC_STD(float, 512, , add); - context->mat_mk[F32].fill = GET_MK_PROC_STD(float, 512, , fill); - context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, 512, , scalar_mul); - context->mat_mk[F32].sub = GET_MK_PROC_STD(float, 512, , sub); - context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, 512, , transpose); - context->mat_mk[F32].fma = GET_MK_PROC_STD(float, 512, , fma); - context->mat_mk[F32].pivot = NULL; + context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, 512, , add); + context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, 512, , fill); + context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, 512, , scalar_mul); + context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, 512, , sub); + context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, 512, , transpose); + context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, 512, , fma); + context->mat_mk[F32].proc_array[Pivot] = NULL; - context->mat_mk[F64].add = GET_MK_PROC_STD(double, 512, , add); - context->mat_mk[F64].fill = GET_MK_PROC_STD(double, 512, , fill); - context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, 512, , scalar_mul); - context->mat_mk[F64].sub = GET_MK_PROC_STD(double, 512, , sub); - context->mat_mk[F64].transpose = NULL; - context->mat_mk[F64].fma = GET_MK_PROC_STD(double, 512, , fma); - context->mat_mk[F64].pivot = NULL; + context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, 512, , add); + context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, 512, , fill); + context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, 512, , scalar_mul); + context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, 512, , sub); + context->mat_mk[F64].proc_array[Transpose] = NULL; + context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, 512, , fma); + context->mat_mk[F64].proc_array[Pivot] = NULL; context->vec_f32_normalize_3 = f32_normalize; } diff --git a/StgSharp.Native/sn-src/mk.fma.pack.c b/StgSharp.Native/sn-src/mk.fma.pack.c new file mode 100644 index 00000000..b348bd2e --- /dev/null +++ b/StgSharp.Native/sn-src/mk.fma.pack.c @@ -0,0 +1,288 @@ +#include "sn_target.h" + +#if SN_IS_ARCH(SN_ARCH_X86_64) + +#include "sn_intrinsic.context.x86.h" + +/* +* pack left matrix data to L4 buffer +* +* CONVENSIONS: +* +* buffer_1 is pointer to original colum, +* offset_1 is the offset to the original column, +* count_1 is the count can be packed +* +* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension, +* and make sure that count_1 will not exceeds lenght of the column +* +* scalar pack carries column length and offset data: +* [column length, empty, empty, empty] +*/ +SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + // for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data + memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2); +} + +SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + size_t i = 0; + for (; i < (count_1 & (~0x1)); i += 2) { + register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); + register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); + register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); + register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); + register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left); + register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left); + register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right); + register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right); + + _mm256_store_ps((float *)&ans[i].y[0], col_0); + _mm256_store_ps((float *)&ans[i].y[1], col_1); + _mm256_store_ps((float *)&ans[i].y[2], col_2); + _mm256_store_ps((float *)&ans[i].y[3], col_3); + } + if (count_1 & 0x1) { + register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); + register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); + register __m256 zero = _mm256_setzero_ps(); + register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero); + register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero); + register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero); + register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero); + + _mm256_store_ps((float *)&ans[i].y[0], col_0); + _mm256_store_ps((float *)&ans[i].y[1], col_1); + _mm256_store_ps((float *)&ans[i].y[2], col_2); + _mm256_store_ps((float *)&ans[i].y[3], col_3); + } +} + +SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + size_t i = 0; + register __m512 ker_0; + register __m512 ker_1; + register __m512 ker_2; + register __m512 ker_3; + register __m512 col_0; + register __m512 col_1; + register __m512 col_2; + register __m512 col_3; + for (; i < (count_1 & (~0x3)); i += 4) { + ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); + ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]); + ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]); + ker_3 = _mm512_load_ps((float *)&right[offset_1 + i].z[3]); + col_0 = _mm512_unpacklo_ps(ker_0, ker_2); + col_1 = _mm512_unpackhi_ps(ker_0, ker_2); + col_2 = _mm512_unpacklo_ps(ker_1, ker_3); + col_3 = _mm512_unpackhi_ps(ker_1, ker_3); + + _mm512_store_ps((float *)&ans[i].z[0], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[1], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2))); + _mm512_store_ps((float *)&ans[i].z[2], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[3], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2))); + } + switch (count_1 & 0x3) { + case 1: + ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); + col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0)); + col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1)); + col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2)); + col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3)); + + _mm512_store_ps((float *)&ans[i].z[0], col_0); + _mm512_store_ps((float *)&ans[i].z[1], col_1); + _mm512_store_ps((float *)&ans[i].z[2], col_2); + _mm512_store_ps((float *)&ans[i].z[3], col_3); + case 2: + ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); + ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]); + col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0)); + col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1)); + col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0)); + col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1)); + + _mm512_store_ps((float *)&ans[i].z[0], col_0); + _mm512_store_ps((float *)&ans[i].z[1], col_1); + _mm512_store_ps((float *)&ans[i].z[2], col_2); + _mm512_store_ps((float *)&ans[i].z[3], col_3); + case 3: + ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); + ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]); + ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]); + ker_3 = _mm512_setzero_ps(); + col_0 = _mm512_unpacklo_ps(ker_0, ker_2); + col_1 = _mm512_unpackhi_ps(ker_0, ker_2); + col_2 = _mm512_unpacklo_ps(ker_1, ker_3); + col_3 = _mm512_unpackhi_ps(ker_1, ker_3); + + _mm512_store_ps((float *)&ans[i].z[0], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[1], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2))); + _mm512_store_ps((float *)&ans[i].z[2], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[3], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2))); + default: + break; + } +} + +/* +* pack right matrix data to L4 buffer +* +* CONVENSIONS: +* +* buffer_1 is pointer to original row, +* offset_1 is the offset to the original row, +* count_1 is the count can be packed +* +* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension, +* and make sure that count_1 will not exceeds lenght of the row +* +* scalar pack carries column length and offset data: +* [column length, empty, empty, empty] +*/ +SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + int col_length = scalar.i32[0]; + for (size_t i = 0; i < count_1; i++) { + ans[i] = right[(offset_1 + i) * col_length]; + } +} + +SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + int col_length = scalar.i32[0]; + size_t i = 0; + for (; i < (count_1 & (~0x1)); i += 2) { + register __m256 ker_0_left = + _mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]); + register __m256 ker_0_right = + _mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]); + register __m256 ker_1_left = + _mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[0]); + register __m256 ker_1_right = + _mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[1]); + register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left); + register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left); + register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right); + register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right); + + _mm256_store_ps((float *)&ans[i].y[0], col_0); + _mm256_store_ps((float *)&ans[i].y[1], col_1); + _mm256_store_ps((float *)&ans[i].y[2], col_2); + _mm256_store_ps((float *)&ans[i].y[3], col_3); + } + if (count_1 & 0x1) { + register __m256 ker_0_left = + _mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]); + register __m256 ker_0_right = + _mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]); + register __m256 zero = _mm256_setzero_ps(); + register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero); + register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero); + register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero); + register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero); + + _mm256_store_ps((float *)&ans[i].y[0], col_0); + _mm256_store_ps((float *)&ans[i].y[1], col_1); + _mm256_store_ps((float *)&ans[i].y[2], col_2); + _mm256_store_ps((float *)&ans[i].y[3], col_3); + } +} + +SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right) +{ + __mk_param_std(RIGHT_ANS_SCALAR, float); + int col_length = scalar.i32[0]; + size_t i = 0; + register __m512 ker_0; + register __m512 ker_1; + register __m512 ker_2; + register __m512 ker_3; + register __m512 col_0; + register __m512 col_1; + register __m512 col_2; + register __m512 col_3; + for (; i < (count_1 & (~0x3)); i += 4) { + ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]); + ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[0]); + ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[0]); + ker_3 = _mm512_load_ps((float *)&right[(offset_1 + i + 3) * col_length].z[0]); + col_0 = _mm512_unpacklo_ps(ker_0, ker_2); + col_1 = _mm512_unpackhi_ps(ker_0, ker_2); + col_2 = _mm512_unpacklo_ps(ker_1, ker_3); + col_3 = _mm512_unpackhi_ps(ker_1, ker_3); + + _mm512_store_ps((float *)&ans[i].z[0], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[1], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2))); + _mm512_store_ps((float *)&ans[i].z[2], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[3], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2))); + } + switch (count_1 & 0x3) { + case 1: + ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i) * col_length].z[0]); + col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0)); + col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1)); + col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2)); + col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3)); + + _mm512_store_ps((float *)&ans[i].z[0], col_0); + _mm512_store_ps((float *)&ans[i].z[1], col_1); + _mm512_store_ps((float *)&ans[i].z[2], col_2); + _mm512_store_ps((float *)&ans[i].z[3], col_3); + case 2: + ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]); + ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]); + col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0)); + col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1)); + col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0)); + col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1)); + + _mm512_store_ps((float *)&ans[i].z[0], col_0); + _mm512_store_ps((float *)&ans[i].z[1], col_1); + _mm512_store_ps((float *)&ans[i].z[2], col_2); + _mm512_store_ps((float *)&ans[i].z[3], col_3); + case 3: + ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]); + ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]); + ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[2]); + ker_3 = _mm512_setzero_ps(); + col_0 = _mm512_unpacklo_ps(ker_0, ker_2); + col_1 = _mm512_unpackhi_ps(ker_0, ker_2); + col_2 = _mm512_unpacklo_ps(ker_1, ker_3); + col_3 = _mm512_unpackhi_ps(ker_1, ker_3); + + _mm512_store_ps((float *)&ans[i].z[0], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[1], + _mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2))); + _mm512_store_ps((float *)&ans[i].z[2], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0))); + _mm512_store_ps((float *)&ans[i].z[3], + _mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2))); + default: + break; + } +} + + +#endif \ No newline at end of file diff --git a/StgSharp.Native/sn-src/mk.fma.x86.c b/StgSharp.Native/sn-src/mk.fma.x86.c index f1e1c1a8..eebdc2c2 100644 --- a/StgSharp.Native/sn-src/mk.fma.x86.c +++ b/StgSharp.Native/sn-src/mk.fma.x86.c @@ -1,5 +1,6 @@ #include "sn_target.h" -/* clang-format off */ +#include + #if SN_IS_ARCH(SN_ARCH_X86_64) // #include "sn_intrinsic.h" @@ -11,89 +12,6 @@ #define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) -// float - -/* -* pack left matrix data to L4 buffer -* scalar pack carries column length and offset data: -* [column length, buffer requires, empty, empty] -*/ -SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left) -{ - __mk_param_std(RIGHT_ANS_SCALAR, float); - // for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data - memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2); -} - -SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left) -{ - __mk_param_std(RIGHT_ANS_SCALAR, float); - size_t i=0; - for(; i <= count_1; i+=2) { - register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); - register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); - register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); - register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); - register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left); - register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left); - register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right); - register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right); - - _mm256_store_ps((float *)&ans[i].y[0], col_0); - _mm256_store_ps((float *)&ans[i].y[1], col_1); - _mm256_store_ps((float *)&ans[i].y[2], col_2); - _mm256_store_ps((float *)&ans[i].y[3], col_3); - } - if(count_1 == i) { - register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); - register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); - register __m256 zero = _mm256_setzero_ps(); - register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero); - register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero); - register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero); - register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero); - - _mm256_store_ps((float *)&ans[i].y[0], col_0); - _mm256_store_ps((float *)&ans[i].y[1], col_1); - _mm256_store_ps((float *)&ans[i].y[2], col_2); - _mm256_store_ps((float *)&ans[i].y[3], col_3); - } -} - -SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left) -{ - __mk_param_std(RIGHT_ANS_SCALAR, float); - size_t i=0; - for(; i <= count_1; i+=2) { - register __m512 ker_0_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); - register __m512 ker_0_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]); - register __m512 ker_1_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]); - register __m512 ker_1_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]); - register __m512 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left); - register __m512 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left); - register __m512 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right); - register __m512 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right); - - _mm256_store_ps((float *)&ans[i].y[0], col_0); - _mm256_store_ps((float *)&ans[i].y[1], col_1); - _mm256_store_ps((float *)&ans[i].y[2], col_2); - _mm256_store_ps((float *)&ans[i].y[3], col_3); - } - if(count_1 == i) { - register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]); - register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]); - register __m256 zero = _mm256_setzero_ps(); - register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero); - register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero); - register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero); - register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero); - - _mm256_store_ps((float *)&ans[i].y[0], col_0); - _mm256_store_ps((float *)&ans[i].y[1], col_1); - _mm256_store_ps((float *)&ans[i].y[2], col_2); - _mm256_store_ps((float *)&ans[i].y[3], col_3); - } -} SN_MK_PROC_DECL_STD(float, sse, , fma) { @@ -178,12 +96,11 @@ SN_MK_PROC_DECL_STD(float, avx, , fma) SN_MK_FMA_F32_AVX2_CYCLE(1); SN_MK_FMA_F32_AVX2_CYCLE(2); SN_MK_FMA_F32_AVX2_CYCLE(3); - } - _mm256_store_ps( (float*)&(ans_cur->y[0]),c0); - _mm256_store_ps( (float*)&(ans_cur->y[1]),c1); - _mm256_store_ps( (float*)&(ans_cur->y[2]),c2); - _mm256_store_ps( (float*)&(ans_cur->y[3]),c3); + _mm256_store_ps((float *)&(ans_cur->y[0]), c0); + _mm256_store_ps((float *)&(ans_cur->y[1]), c1); + _mm256_store_ps((float *)&(ans_cur->y[2]), c2); + _mm256_store_ps((float *)&(ans_cur->y[3]), c3); } SN_MK_PROC_DECL_STD(float, avx, _fma, fma) @@ -195,7 +112,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]); register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]); - for (size_t z = 0; z < count_0; z+=2) { + for (size_t z = 0; z < count_0; z += 2) { const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z; @@ -213,15 +130,13 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) SN_MK_FMA_F32_AVX2_FMA_CYCLE(1); SN_MK_FMA_F32_AVX2_FMA_CYCLE(2); SN_MK_FMA_F32_AVX2_FMA_CYCLE(3); - } - _mm256_store_ps( (float*)&(ans_cur->y[0]),c0); - _mm256_store_ps( (float*)&(ans_cur->y[1]),c1); - _mm256_store_ps( (float*)&(ans_cur->y[2]),c2); - _mm256_store_ps( (float*)&(ans_cur->y[3]),c3); + _mm256_store_ps((float *)&(ans_cur->y[0]), c0); + _mm256_store_ps((float *)&(ans_cur->y[1]), c1); + _mm256_store_ps((float *)&(ans_cur->y[2]), c2); + _mm256_store_ps((float *)&(ans_cur->y[3]), c3); } - SN_MK_PROC_DECL_STD(float, 512, , fma) { __mk_param_std(LEFT_RIGHT_ANS, float); @@ -231,7 +146,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]); register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]); - for (size_t z = 0; z < count_0; z+=4) { + for (size_t z = 0; z < count_0; z += 4) { const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z; @@ -249,15 +164,13 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) SN_MK_FMA_F32_512_CYCLE(1); SN_MK_FMA_F32_512_CYCLE(2); SN_MK_FMA_F32_512_CYCLE(3); - } - _mm512_store_ps((float *)&ans_cur->z[0], c0); - _mm512_store_ps((float *)&ans_cur->z[1], c1); - _mm512_store_ps((float *)&ans_cur->z[2], c2); - _mm512_store_ps((float *)&ans_cur->z[3], c3); + _mm512_store_ps((float *)&ans_cur->z[0], c0); + _mm512_store_ps((float *)&ans_cur->z[1], c1); + _mm512_store_ps((float *)&ans_cur->z[2], c2); + _mm512_store_ps((float *)&ans_cur->z[3], c3); } - SN_MK_PROC_DECL_STD(double, sse, , fma) { __mk_param_std(LEFT_RIGHT_ANS, double); @@ -436,7 +349,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]); register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]); - for (size_t z = 0; z < count_0; z+=2) { + for (size_t z = 0; z < count_0; z += 2) { SN_MAT_KERNEL_LOCAL const *right_base = right + z; SN_MAT_KERNEL_LOCAL const *left_base = left + z; @@ -449,7 +362,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]); register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]); register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]); - + SN_MK_FMA_F64_AVX2_CYCLE(0); SN_MK_FMA_F64_AVX2_CYCLE(1); SN_MK_FMA_F64_AVX2_CYCLE(2);