From 7bcb460a3994dda40f24cae0044b5a36f4f16515 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 9 Dec 2025 18:07:53 +0800 Subject: [PATCH] fma code added --- StgSharp.Common/Mathematics/Graphic/Camera.cs | 4 +- .../Mathematics/Graphic/Matrix44.cs | 100 +++------- StgSharp.Common/Mathematics/Graphic/Vec4.cs | 22 ++- StgSharp.Native/sn-head/sn_intrinsic.h | 39 ++-- StgSharp.Native/sn-head/sn_matkernel.h | 16 +- StgSharp.Native/sn-src/k_fma.c | 177 +++++++++++++++--- StgSharp.Native/sn-src/k_panel.c | 4 +- 7 files changed, 230 insertions(+), 132 deletions(-) diff --git a/StgSharp.Common/Mathematics/Graphic/Camera.cs b/StgSharp.Common/Mathematics/Graphic/Camera.cs index 1ea2c660..93683735 100644 --- a/StgSharp.Common/Mathematics/Graphic/Camera.cs +++ b/StgSharp.Common/Mathematics/Graphic/Camera.cs @@ -153,10 +153,10 @@ namespace StgSharp.Mathematics.Graphic if (direction.GetLength() == 0) { // deadlock - throw new ArgumentException(); + throw new ArgumentException("Direction bust is not zero."); } if (Vec3.IsParallel(direction, up)) { - throw new ArgumentException(); + throw new ArgumentException("Direction and UP is on on same way"); } _target = direction; this.up = up; diff --git a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs index 48c55340..ebe17a43 100644 --- a/StgSharp.Common/Mathematics/Graphic/Matrix44.cs +++ b/StgSharp.Common/Mathematics/Graphic/Matrix44.cs @@ -36,29 +36,24 @@ using System.Runtime.InteropServices; namespace StgSharp.Mathematics.Graphic { - [CollectionBuilder(builderType:typeof(Matrix44), methodName: nameof(Matrix44.FromSpan))] [StructLayout(LayoutKind.Explicit, Size = (2 * 16 * sizeof(float)) + sizeof(bool), Pack = 16)] public unsafe struct Matrix44 : IEquatable, IMatrix, IEnumerable { - [FieldOffset(2 * 64 * sizeof(float))] internal bool isTransposed; [FieldOffset(0 * sizeof(float))] internal Column column; [FieldOffset(0)] internal ColumnSet4 mat; - [FieldOffset(16 * sizeof(float))] internal ColumnSet4 transpose; [FieldOffset(0)] internal MatrixKernel kernel; internal Matrix44(ColumnSet4 mat) { Unsafe.SkipInit(out this); - isTransposed = false; this.mat = mat; } internal Matrix44(Vector4 vec0, Vector4 vec1, Vector4 vec2, Vector4 vec3) { Unsafe.SkipInit(out this); - isTransposed = false; mat.colum0 = vec0; mat.colum1 = vec1; mat.colum2 = vec2; @@ -68,7 +63,6 @@ namespace StgSharp.Mathematics.Graphic public Matrix44() { Unsafe.SkipInit(out this); - isTransposed = false; } public unsafe Matrix44( @@ -90,7 +84,6 @@ namespace StgSharp.Mathematics.Graphic float a33) { Unsafe.SkipInit(out this); - isTransposed = false; mat.colum0 = new Vector4(a00, a10, a20, a30); mat.colum1 = new Vector4(a01, a11, a21, a31); mat.colum2 = new Vector4(a02, a12, a22, a32); @@ -101,7 +94,7 @@ namespace StgSharp.Mathematics.Graphic { get { - #if DEBUG +#if DEBUG if (rowNum is > 3 or < 0) { throw new ArgumentOutOfRangeException(nameof(columNum)); @@ -109,17 +102,12 @@ namespace StgSharp.Mathematics.Graphic if (columNum is > 3 or < 0) { throw new ArgumentOutOfRangeException(nameof(columNum)); } - #endif - InternalTranspose(); - fixed (float* p = &transpose.m00) - { - ulong pBit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum; - return *(float*)pBit; - } +#endif + return kernel[rowNum + 4 * columNum]; } set { - #if DEBUG +#if DEBUG if (rowNum is > 3 or < 0) { throw new ArgumentOutOfRangeException(nameof(columNum)); @@ -127,14 +115,8 @@ namespace StgSharp.Mathematics.Graphic if (columNum is > 3 or < 0) { throw new ArgumentOutOfRangeException(nameof(columNum)); } - #endif - InternalTranspose(); - fixed (float* p = &transpose.m00) - { - ulong pbit = (ulong)p + (ulong)sizeof(Vector4) * (ulong)rowNum + sizeof(float) * (ulong)columNum; - *(float*)pbit = value; - } - isTransposed = false; +#endif + kernel[rowNum + 4 * columNum] = value; } } @@ -142,8 +124,9 @@ namespace StgSharp.Mathematics.Graphic { get { - InternalTranspose(); - return new Matrix44(transpose); + Matrix44 transpose = new(); + InternalTranspose(ref transpose); + return transpose; } } @@ -176,41 +159,17 @@ namespace StgSharp.Mathematics.Graphic return mat.Equals(other.mat); } - public static Matrix44 FromSpan(ReadOnlySpan rowSpan) - { - Matrix44 mat = new Matrix44(); - - mat.transpose.colum0 = rowSpan[0].vec; - mat.transpose.colum1 = rowSpan[1].vec; - mat.transpose.colum2 = rowSpan[2].vec; - mat.transpose.colum3 = rowSpan[3].vec; - - NativeIntrinsic.Intrinsic.f32_buffer_transpose(&mat.transpose, &mat.mat); - mat.isTransposed = false; - return mat; - } - public override int GetHashCode() { return HashCode.Combine(mat.colum0.GetHashCode(), mat.colum1.GetHashCode(), mat.colum2.GetHashCode(), mat.colum3.GetHashCode()); } - public override string ToString() - { - InternalTranspose(); - return $"{transpose.colum0}{transpose.colum1}{transpose.colum2}{transpose.colum3}"; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal unsafe void InternalTranspose() + internal unsafe void InternalTranspose(ref Matrix44 transpose) { - if (!isTransposed) - { - fixed (ColumnSet4* source = &mat, target = &transpose) { - NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target); - } - isTransposed = true; + fixed (MatrixKernel* source = &this.kernel, target = &transpose.kernel) { + NativeIntrinsic.Intrinsic.f32_buffer_transpose(source, target); } } @@ -237,23 +196,23 @@ namespace StgSharp.Mathematics.Graphic [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Matrix44 operator *(Matrix44 left, Matrix44 right) { - left.InternalTranspose(); - return new Matrix44(Vector4.Dot(left.transpose.colum0, right.mat.colum0), - Vector4.Dot(left.transpose.colum0, right.mat.colum1), - Vector4.Dot(left.transpose.colum0, right.mat.colum2), - Vector4.Dot(left.transpose.colum0, right.mat.colum3), - Vector4.Dot(left.transpose.colum1, right.mat.colum0), - Vector4.Dot(left.transpose.colum1, right.mat.colum1), - Vector4.Dot(left.transpose.colum1, right.mat.colum2), - Vector4.Dot(left.transpose.colum1, right.mat.colum3), - Vector4.Dot(left.transpose.colum2, right.mat.colum0), - Vector4.Dot(left.transpose.colum2, right.mat.colum1), - Vector4.Dot(left.transpose.colum2, right.mat.colum2), - Vector4.Dot(left.transpose.colum2, right.mat.colum3), - Vector4.Dot(left.transpose.colum3, right.mat.colum0), - Vector4.Dot(left.transpose.colum3, right.mat.colum1), - Vector4.Dot(left.transpose.colum3, right.mat.colum2), - Vector4.Dot(left.transpose.colum3, right.mat.colum3)); + Matrix44 transpose = left.Transpose; + return new Matrix44(Vector4.Dot(transpose.mat.colum0, right.mat.colum0), + Vector4.Dot(transpose.mat.colum0, right.mat.colum1), + Vector4.Dot(transpose.mat.colum0, right.mat.colum2), + Vector4.Dot(transpose.mat.colum0, right.mat.colum3), + Vector4.Dot(transpose.mat.colum1, right.mat.colum0), + Vector4.Dot(transpose.mat.colum1, right.mat.colum1), + Vector4.Dot(transpose.mat.colum1, right.mat.colum2), + Vector4.Dot(transpose.mat.colum1, right.mat.colum3), + Vector4.Dot(transpose.mat.colum2, right.mat.colum0), + Vector4.Dot(transpose.mat.colum2, right.mat.colum1), + Vector4.Dot(transpose.mat.colum2, right.mat.colum2), + Vector4.Dot(transpose.mat.colum2, right.mat.colum3), + Vector4.Dot(transpose.mat.colum3, right.mat.colum0), + Vector4.Dot(transpose.mat.colum3, right.mat.colum1), + Vector4.Dot(transpose.mat.colum3, right.mat.colum2), + Vector4.Dot(transpose.mat.colum3, right.mat.colum3)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -291,3 +250,4 @@ namespace StgSharp.Mathematics.Graphic } } + diff --git a/StgSharp.Common/Mathematics/Graphic/Vec4.cs b/StgSharp.Common/Mathematics/Graphic/Vec4.cs index 1629f1ec..d3e9c4a8 100644 --- a/StgSharp.Common/Mathematics/Graphic/Vec4.cs +++ b/StgSharp.Common/Mathematics/Graphic/Vec4.cs @@ -140,13 +140,13 @@ namespace StgSharp.Mathematics.Graphic public static Vec4 FromSpan(ReadOnlySpan span) { - return new Vec4 - { - X = span[0], - Y = span[1], - Z = span[2], - W = span[3] - }; + if (span.Length < 4) { + throw new ArgumentException("Span length must be at least 4.", nameof(span)); + } + + // Read 4 floats as a Vector4 in one operation without per-element copy. + Vector4 v = MemoryMarshal.Read(MemoryMarshal.AsBytes(span)); + return new Vec4(v); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -185,9 +185,11 @@ namespace StgSharp.Mathematics.Graphic [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vec4 operator *(Matrix44 mat, Vec4 vec) { - mat.InternalTranspose(); - return new Vec4(Vector4.Dot(mat.transpose.colum0, vec.vec), Vector4.Dot(mat.transpose.colum1, vec.vec), - Vector4.Dot(mat.transpose.colum2, vec.vec), Vector4.Dot(mat.transpose.colum3, vec.vec)); + Matrix44 transpose = mat.Transpose; + return new Vec4(Vector4.Dot(transpose.mat.colum0, vec.vec), + Vector4.Dot(transpose.mat.colum1, vec.vec), + Vector4.Dot(transpose.mat.colum2, vec.vec), + Vector4.Dot(transpose.mat.colum3, vec.vec)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index e16df462..72a19eed 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -141,22 +141,26 @@ typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pac #define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch -#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ - MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ - MAT_PANEL##T_arch##(T_type) *restrict ans) -#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ - MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ - MAT_PANEL##T_arch##(T_type) *restrict ans) -#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ - MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ - MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) -#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ - INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ - MAT_PANEL##T_arch##(T_type) const *left, MAT_PANEL##T_arch##(T_type) const *right, \ - MAT_PANEL##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \ + T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \ + MAT_PANEL_##T_arch##(T_type) const *right, \ + MAT_PANEL_##T_arch##(T_type) *restrict ans) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \ + T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \ + MAT_PANEL_##T_arch##(T_type) const *right, \ + MAT_PANEL_##T_arch##(T_type) *restrict ans) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL_##T_arch##(T_type) const *left, \ + MAT_PANEL_##T_arch##(T_type) const *right, \ + MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) +#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \ + MAT_PANEL_##T_arch##(T_type) const *left, \ + MAT_PANEL_##T_arch##(T_type) const *right, \ + MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar) #pragma endregion @@ -219,7 +223,8 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill); #pragma region fma -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma); +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma); +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma); #pragma endregion diff --git a/StgSharp.Native/sn-head/sn_matkernel.h b/StgSharp.Native/sn-head/sn_matkernel.h index 857b7af7..6c1b12fc 100644 --- a/StgSharp.Native/sn-head/sn_matkernel.h +++ b/StgSharp.Native/sn-head/sn_matkernel.h @@ -67,15 +67,19 @@ typedef MAT_KERNEL(uint64_t) sn_zero_kernel_u64; /* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4. Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128). Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */ -#define MAT_PANEL_SSE(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4) +#define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, 4) /* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8. Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit. Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */ -#define MAT_PANEL_AVX(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8) +#define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8) +/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ=8. + Columns: each column stores eight elements via two 256-bit chunks when T is 32-bit. + Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */ +#define MAT_PANEL_avx_fma(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, 8) /* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T. - Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps double-wide tiles. + Columns: each column expands to ceil(Q/lanes) ZMM chunks; default MINQ=8 keeps doublse-wide tiles. Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */ -#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8) +#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, 8) /* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */ #ifdef __cplusplus @@ -113,7 +117,7 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL; int col_index, int row_index) /* Declarations for AVX float variant */ -DECLARE_BUILD_PANEL(AVX, float); -DECLARE_STORE_PANEL(AVX, float); +DECLARE_BUILD_PANEL(avx, float); +DECLARE_STORE_PANEL(avx, float); #endif /* SN_MATKERNEL */ diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index a600e7cf..d5e8510c 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -1,35 +1,162 @@ #include "sn_intrinsic.h" +#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) + // ans = left * right + ans // right is not transposed -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, fma) +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma) { - register __m128 source0 = _mm_load_ps(&left->xmm[0]); - register __m128 source1 = _mm_load_ps(&left->xmm[1]); - register __m128 source2 = _mm_load_ps(&left->xmm[2]); - register __m128 source3 = _mm_load_ps(&left->xmm[3]); + const __m128 a0 = PANEL_COL_VEC((*left), 0, 0); + const __m128 a1 = PANEL_COL_VEC((*left), 1, 0); + const __m128 a2 = PANEL_COL_VEC((*left), 2, 0); + const __m128 a3 = PANEL_COL_VEC((*left), 3, 0); - register __m128 ans0 = _mm_load_ps(&ans->xmm[0]); - register __m128 ans1 = _mm_load_ps(&ans->xmm[1]); - register __m128 ans2 = _mm_load_ps(&ans->xmm[2]); - register __m128 ans3 = _mm_load_ps(&ans->xmm[3]); + __m128 c0 = PANEL_COL_VEC((*ans), 0, 0); + __m128 c1 = PANEL_COL_VEC((*ans), 1, 0); + __m128 c2 = PANEL_COL_VEC((*ans), 2, 0); + __m128 c3 = PANEL_COL_VEC((*ans), 3, 0); - register __m128 broadcast; - for (size_t i = 0; i < 4; i++) { - broadcast = _mm_load_ps(&right->xmm[i]); + __m128 bcol; - ans0 = _mm_add_ps(ans0, - _mm_mul_ps(source0, _mm_shuffle_ps(broadcast, broadcast, 0x00))); - ans1 = _mm_add_ps(ans1, - _mm_mul_ps(source1, _mm_shuffle_ps(broadcast, broadcast, 0x55))); - ans2 = _mm_add_ps(ans2, - _mm_mul_ps(source2, _mm_shuffle_ps(broadcast, broadcast, 0xAA))); - ans3 = _mm_add_ps(ans3, - _mm_mul_ps(source3, _mm_shuffle_ps(broadcast, broadcast, 0xFF))); - } + /* Column 0 */ + bcol = PANEL_COL_VEC((*right), 0, 0); + c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); + c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); + c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); + c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); - _mm_store_ps(&ans->xmm[0], ans0); - _mm_store_ps(&ans->xmm[1], ans1); - _mm_store_ps(&ans->xmm[2], ans2); - _mm_store_ps(&ans->xmm[3], ans3); + /* Column 1 */ + bcol = PANEL_COL_VEC((*right), 1, 0); + c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); + c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); + c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); + c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); + + /* Column 2 */ + bcol = PANEL_COL_VEC((*right), 2, 0); + c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); + c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); + c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); + c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); + + /* Column 3 */ + bcol = PANEL_COL_VEC((*right), 3, 0); + c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0)))); + c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1)))); + c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2)))); + c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3)))); + + PANEL_COL_VEC((*ans), 0, 0) = c0; + PANEL_COL_VEC((*ans), 1, 0) = c1; + PANEL_COL_VEC((*ans), 2, 0) = c2; + PANEL_COL_VEC((*ans), 3, 0) = c3; +} + +// ans = left * right + ans +// right is not transposed +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma) +{ + static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 }; + + for (int col = 0; col < 8; ++col) { + register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]); + register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]); + register __m256i idx = _mm256_setzero_si256(); + register __m256i increament = _mm256_load_si256(&idx0); + + register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]); + register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]); + register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]); + register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]); + + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + + a0 = _mm256_load_ps((float *)&left->vec[4][0]); + a1 = _mm256_load_ps((float *)&left->vec[5][0]); + a2 = _mm256_load_ps((float *)&left->vec[6][0]); + a3 = _mm256_load_ps((float *)&left->vec[7][0]); + + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a0, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a1, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a2, _mm256_permutevar8x32_ps(bcol, idx))); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_add_ps(ccol, _mm256_mul_ps(a3, _mm256_permutevar8x32_ps(bcol, idx))); + + _mm256_store_ps((float *)&ans->vec[col][0], ccol); + } +} + +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma) +{ + const __m512i ones = _mm512_set1_epi32(1); + + for (int col = 0; col < 8; ++col) { + register __m512 bcol = _mm512_load_ps((float *)&right->vec[col][0]); + register __m512 ccol = _mm512_load_ps((float *)&ans->vec[col][0]); + register __m512i idx = _mm512_setzero_si512(); + register __m512i increament = _mm512_load_si512(&ones); + + register __m512 a0 = _mm512_load_ps((float *)&left->vec[0][0]); + register __m512 a1 = _mm512_load_ps((float *)&left->vec[1][0]); + register __m512 a2 = _mm512_load_ps((float *)&left->vec[2][0]); + register __m512 a3 = _mm512_load_ps((float *)&left->vec[3][0]); + register __m512 a4 = _mm512_load_ps((float *)&left->vec[4][0]); + register __m512 a5 = _mm512_load_ps((float *)&left->vec[5][0]); + register __m512 a6 = _mm512_load_ps((float *)&left->vec[6][0]); + register __m512 a7 = _mm512_load_ps((float *)&left->vec[7][0]); + + ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + + a0 = _mm512_load_ps((float *)&left->vec[8][0]); + a1 = _mm512_load_ps((float *)&left->vec[9][0]); + a2 = _mm512_load_ps((float *)&left->vec[10][0]); + a3 = _mm512_load_ps((float *)&left->vec[11][0]); + a4 = _mm512_load_ps((float *)&left->vec[12][0]); + a5 = _mm512_load_ps((float *)&left->vec[13][0]); + a6 = _mm512_load_ps((float *)&left->vec[14][0]); + a7 = _mm512_load_ps((float *)&left->vec[15][0]); + + ccol = _mm512_fmadd_ps(a0, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a1, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a2, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a3, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a4, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a5, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a6, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + ccol = _mm512_fmadd_ps(a7, _mm512_permutexvar_ps(idx, bcol), ccol); + idx = _mm512_add_epi32(idx, increament); + + _mm512_store_ps((float *)&ans->vec[col][0], ccol); + } } diff --git a/StgSharp.Native/sn-src/k_panel.c b/StgSharp.Native/sn-src/k_panel.c index ae59bea7..baa40211 100644 --- a/StgSharp.Native/sn-src/k_panel.c +++ b/StgSharp.Native/sn-src/k_panel.c @@ -1,6 +1,6 @@ #include "sn_matkernel.h" -DECLARE_BUILD_PANEL(AVX, float) +DECLARE_BUILD_PANEL(avx, float) { const int Q = 8; /* fixed for AVX float panel */ @@ -41,7 +41,7 @@ DECLARE_BUILD_PANEL(AVX, float) } } -DECLARE_STORE_PANEL(AVX, float) +DECLARE_STORE_PANEL(avx, float) { const int Q = 8; /* fixed for AVX float panel */