youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

left and right use diff pack kernel

Browse Source
This commit is contained in:
Youmu Liu
2026-06-12 18:25:04 +08:00
parent 06a119cc6f
commit 2741b34583
6 changed files with 390 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Mathematics/Internal/IntrinsicContext.cs
+3 -1
View File
@@ -42,7 +42,9 @@ namespace StgSharp.Mathematics.Internal
Transpose = 4, Transpose = 4,
ClearPanel = 5, ClearPanel = 5,
Fma = 6, Fma = 6,
Pivot = 8, PackFmaLeft = 7,
PackFmaRight = 8,
Pivot = 9,
} }
StgSharp.Native/sn-head/sn_intrinsic.context.x86.h
+24 -2
View File
@@ -175,17 +175,39 @@ SN_MK_PROC_DECL_STD(float, sse, , quality);
* *
* CONVENSIONS: * CONVENSIONS:
* *
* buffer_1 is pointer to original colum, offset_1 is the offset to the original column, count_1 is the count can be packed * buffer_1 is pointer to original colum,
* offset_1 is the offset to the original column,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension, * C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the column * and make sure that count_1 will not exceeds lenght of the column
* *
* scalar pack carries column length and offset data: * scalar pack carries column length and offset data:
* [column length, offset, buffer requires, empty] * [column length, empty, empty, empty]
*/ */
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left); SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left); SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left); SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
/*
* pack right matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original row,
* offset_1 is the offset to the original row,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the row
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, sse, , fma); SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma); SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma); SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
StgSharp.Native/sn-head/sn_intrinsic.h
+15 -14
View File
@@ -160,23 +160,11 @@ typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length
#pragma endregion #pragma endregion
typedef struct mat_mk_intrinsic { typedef struct mat_mk_intrinsic {
UNI_MK_PROC add; UNI_MK_PROC proc_array[10];
UNI_MK_PROC fill;
UNI_MK_PROC scalar_mul;
UNI_MK_PROC sub;
UNI_MK_PROC transpose;
UNI_MK_PROC fma;
UNI_MK_PROC pivot;
} mat_mk_intrinsic; } mat_mk_intrinsic;
typedef struct mat_sk_intrinsic { typedef struct mat_sk_intrinsic {
UNI_SK_PROC add; UNI_SK_PROC proc_array[10];
UNI_SK_PROC fill;
UNI_SK_PROC scalar_mul;
UNI_SK_PROC sub;
UNI_SK_PROC transpose;
UNI_SK_PROC fma;
UNI_SK_PROC pivot;
} mat_sk_intrinsic; } mat_sk_intrinsic;
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync. // Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
@@ -188,6 +176,19 @@ typedef struct sn_intrinsic {
mat_sk_intrinsic mat_sk[0x0a]; mat_sk_intrinsic mat_sk[0x0a];
} sn_intrinsic; } sn_intrinsic;
typedef enum mat_proc_index {
Add = 0,
Fill = 1,
ScalarMul = 2,
Sub = 3,
Transpose = 4,
ClearPanel = 5,
Fma = 6,
PackFmaLeft = 7,
PackFmaRight = 8,
Pivot = 9,
} mat_proc_index;
typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask; typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask;
#endif #endif
StgSharp.Native/sn-src/load_intrinsic.x86.c
+42 -42
View File
@@ -44,21 +44,21 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
context->city_hash_simplify = city_hash_simplify_sse; context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse; context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].add = GET_MK_PROC_STD(float, sse, , add); context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, sse, , add);
context->mat_mk[F32].fill = GET_MK_PROC_STD(float, sse, , fill); context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, sse, , fill);
context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, sse, , scalar_mul); context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, sse, , scalar_mul);
context->mat_mk[F32].sub = GET_MK_PROC_STD(float, sse, , sub); context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, sse, , sub);
context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, sse, , transpose); context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, sse, , transpose);
context->mat_mk[F32].fma = GET_MK_PROC_STD(float, sse, , fma); context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, sse, , fma);
context->mat_mk[F32].pivot = NULL; context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].add = GET_MK_PROC_STD(double, sse, , add); context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, sse, , add);
context->mat_mk[F64].fill = GET_MK_PROC_STD(double, sse, , fill); context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, sse, , fill);
context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, sse, , scalar_mul); context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, sse, , scalar_mul);
context->mat_mk[F64].sub = GET_MK_PROC_STD(double, sse, , sub); context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, sse, , sub);
context->mat_mk[F64].transpose = NULL; context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].fma = GET_MK_PROC_STD(double, sse, , fma); context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, sse, , fma);
context->mat_mk[F64].pivot = NULL; context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize; context->vec_f32_normalize_3 = f32_normalize;
} }
@@ -71,23 +71,23 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
context->city_hash_simplify = city_hash_simplify_sse; context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse; context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].add = GET_MK_PROC_STD(float, avx, , add); context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, avx, , add);
context->mat_mk[F32].fill = GET_MK_PROC_STD(float, avx, , fill); context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, avx, , fill);
context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, avx, , scalar_mul); context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, avx, , scalar_mul);
context->mat_mk[F32].sub = GET_MK_PROC_STD(float, avx, , sub); context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, avx, , sub);
context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, avx, , transpose); context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, avx, , transpose);
context->mat_mk[F32].fma = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) : context->mat_mk[F32].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) :
GET_MK_PROC_STD(float, avx, , fma); GET_MK_PROC_STD(float, avx, , fma);
context->mat_mk[F32].pivot = NULL; context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].add = GET_MK_PROC_STD(double, avx, , add); context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, avx, , add);
context->mat_mk[F64].fill = GET_MK_PROC_STD(double, avx, , fill); context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, avx, , fill);
context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, avx, , scalar_mul); context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, avx, , scalar_mul);
context->mat_mk[F64].sub = GET_MK_PROC_STD(double, avx, , sub); context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, avx, , sub);
context->mat_mk[F64].transpose = NULL; context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].fma = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) : context->mat_mk[F64].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) :
GET_MK_PROC_STD(double, avx, , fma); GET_MK_PROC_STD(double, avx, , fma);
context->mat_mk[F64].pivot = NULL; context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize; context->vec_f32_normalize_3 = f32_normalize;
} }
@@ -97,21 +97,21 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
context->city_hash_simplify = city_hash_simplify_sse; context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse; context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].add = GET_MK_PROC_STD(float, 512, , add); context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, 512, , add);
context->mat_mk[F32].fill = GET_MK_PROC_STD(float, 512, , fill); context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, 512, , fill);
context->mat_mk[F32].scalar_mul = GET_MK_PROC_STD(float, 512, , scalar_mul); context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, 512, , scalar_mul);
context->mat_mk[F32].sub = GET_MK_PROC_STD(float, 512, , sub); context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, 512, , sub);
context->mat_mk[F32].transpose = GET_MK_PROC_STD(float, 512, , transpose); context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, 512, , transpose);
context->mat_mk[F32].fma = GET_MK_PROC_STD(float, 512, , fma); context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, 512, , fma);
context->mat_mk[F32].pivot = NULL; context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].add = GET_MK_PROC_STD(double, 512, , add); context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, 512, , add);
context->mat_mk[F64].fill = GET_MK_PROC_STD(double, 512, , fill); context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, 512, , fill);
context->mat_mk[F64].scalar_mul = GET_MK_PROC_STD(double, 512, , scalar_mul); context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, 512, , scalar_mul);
context->mat_mk[F64].sub = GET_MK_PROC_STD(double, 512, , sub); context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, 512, , sub);
context->mat_mk[F64].transpose = NULL; context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].fma = GET_MK_PROC_STD(double, 512, , fma); context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, 512, , fma);
context->mat_mk[F64].pivot = NULL; context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize; context->vec_f32_normalize_3 = f32_normalize;
} }
StgSharp.Native/sn-src/mk.fma.pack.c
+288
View File
@@ -0,0 +1,288 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.context.x86.h"
/*
* pack left matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original colum,
* offset_1 is the offset to the original column,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the column
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_load_ps((float *)&right[offset_1 + i].z[3]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
/*
* pack right matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original row,
* offset_1 is the offset to the original row,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the row
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
for (size_t i = 0; i < count_1; i++) {
ans[i] = right[(offset_1 + i) * col_length];
}
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 ker_1_left =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[0]);
register __m256 ker_1_right =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[0]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[0]);
ker_3 = _mm512_load_ps((float *)&right[(offset_1 + i + 3) * col_length].z[0]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i) * col_length].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
#endif
StgSharp.Native/sn-src/mk.fma.x86.c
+18 -105
View File
@@ -1,5 +1,6 @@
#include "sn_target.h" #include "sn_target.h"
/* clang-format off */ #include <immintrin.h>
#if SN_IS_ARCH(SN_ARCH_X86_64) #if SN_IS_ARCH(SN_ARCH_X86_64)
// #include "sn_intrinsic.h" // #include "sn_intrinsic.h"
@@ -11,89 +12,6 @@
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) #define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
// float
/*
* pack left matrix data to L4 buffer
* scalar pack carries column length and offset data:
* [column length, buffer requires, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i=0;
for(; i <= count_1; i+=2) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if(count_1 == i) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i=0;
for(; i <= count_1; i+=2) {
register __m512 ker_0_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
register __m512 ker_0_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
register __m512 ker_1_left = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
register __m512 ker_1_right = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
register __m512 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m512 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m512 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m512 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if(count_1 == i) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, sse, , fma) SN_MK_PROC_DECL_STD(float, sse, , fma)
{ {
@@ -178,12 +96,11 @@ SN_MK_PROC_DECL_STD(float, avx, , fma)
SN_MK_FMA_F32_AVX2_CYCLE(1); SN_MK_FMA_F32_AVX2_CYCLE(1);
SN_MK_FMA_F32_AVX2_CYCLE(2); SN_MK_FMA_F32_AVX2_CYCLE(2);
SN_MK_FMA_F32_AVX2_CYCLE(3); SN_MK_FMA_F32_AVX2_CYCLE(3);
} }
_mm256_store_ps( (float*)&(ans_cur->y[0]),c0); _mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps( (float*)&(ans_cur->y[1]),c1); _mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps( (float*)&(ans_cur->y[2]),c2); _mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps( (float*)&(ans_cur->y[3]),c3); _mm256_store_ps((float *)&(ans_cur->y[3]), c3);
} }
SN_MK_PROC_DECL_STD(float, avx, _fma, fma) SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
@@ -195,7 +112,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]); register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]); register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z+=2) { for (size_t z = 0; z < count_0; z += 2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -213,15 +130,13 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
SN_MK_FMA_F32_AVX2_FMA_CYCLE(1); SN_MK_FMA_F32_AVX2_FMA_CYCLE(1);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(2); SN_MK_FMA_F32_AVX2_FMA_CYCLE(2);
SN_MK_FMA_F32_AVX2_FMA_CYCLE(3); SN_MK_FMA_F32_AVX2_FMA_CYCLE(3);
} }
_mm256_store_ps( (float*)&(ans_cur->y[0]),c0); _mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps( (float*)&(ans_cur->y[1]),c1); _mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps( (float*)&(ans_cur->y[2]),c2); _mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps( (float*)&(ans_cur->y[3]),c3); _mm256_store_ps((float *)&(ans_cur->y[3]), c3);
} }
SN_MK_PROC_DECL_STD(float, 512, , fma) SN_MK_PROC_DECL_STD(float, 512, , fma)
{ {
__mk_param_std(LEFT_RIGHT_ANS, float); __mk_param_std(LEFT_RIGHT_ANS, float);
@@ -231,7 +146,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]); register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]); register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < count_0; z+=4) { for (size_t z = 0; z < count_0; z += 4) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z; const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z; const SN_MAT_KERNEL_LOCAL *b_base = right + z;
@@ -249,15 +164,13 @@ SN_MK_PROC_DECL_STD(float, 512, , fma)
SN_MK_FMA_F32_512_CYCLE(1); SN_MK_FMA_F32_512_CYCLE(1);
SN_MK_FMA_F32_512_CYCLE(2); SN_MK_FMA_F32_512_CYCLE(2);
SN_MK_FMA_F32_512_CYCLE(3); SN_MK_FMA_F32_512_CYCLE(3);
} }
_mm512_store_ps((float *)&ans_cur->z[0], c0); _mm512_store_ps((float *)&ans_cur->z[0], c0);
_mm512_store_ps((float *)&ans_cur->z[1], c1); _mm512_store_ps((float *)&ans_cur->z[1], c1);
_mm512_store_ps((float *)&ans_cur->z[2], c2); _mm512_store_ps((float *)&ans_cur->z[2], c2);
_mm512_store_ps((float *)&ans_cur->z[3], c3); _mm512_store_ps((float *)&ans_cur->z[3], c3);
} }
SN_MK_PROC_DECL_STD(double, sse, , fma) SN_MK_PROC_DECL_STD(double, sse, , fma)
{ {
__mk_param_std(LEFT_RIGHT_ANS, double); __mk_param_std(LEFT_RIGHT_ANS, double);
@@ -436,7 +349,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]); register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]); register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < count_0; z+=2) { for (size_t z = 0; z < count_0; z += 2) {
SN_MAT_KERNEL_LOCAL const *right_base = right + z; SN_MAT_KERNEL_LOCAL const *right_base = right + z;
SN_MAT_KERNEL_LOCAL const *left_base = left + z; SN_MAT_KERNEL_LOCAL const *left_base = left + z;
@@ -449,7 +362,7 @@ SN_MK_PROC_DECL_STD(double, 512, , fma)
register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]); register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]); register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]); register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0); SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1); SN_MK_FMA_F64_AVX2_CYCLE(1);
SN_MK_FMA_F64_AVX2_CYCLE(2); SN_MK_FMA_F64_AVX2_CYCLE(2);