From 88af8d8957efbf288c45d0da922159d116bafc9b Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 7 Jan 2026 18:06:36 +0800 Subject: [PATCH] better kernel fma --- StgSharp.Common/Environment/Platform.cs | 77 +++++-- .../HighPerformance/MatrixPanel.cs | 59 +++-- StgSharp.Common/Internal/Intrinsic/SIMDID.cs | 51 +++++ .../Intrinsic/intrinsicContextProvider.cs | 29 +-- StgSharp.Common/Mathematics/Linear/Matrix.cs | 7 +- StgSharp.Common/Mathematics/Scalar/Random.cs | 12 +- .../Numeric/MatrixCompute.Float.cs | 23 +- StgSharp.Mathematics/Numeric/MatrixCompute.cs | 9 +- StgSharp.Native/sn-src/cpuid.x86.c | 2 +- StgSharp.Native/sn-src/m.fma.x86.c | 207 +++++++++++++----- StgSharpDebug/Program.cs | 20 +- 11 files changed, 347 insertions(+), 149 deletions(-) create mode 100644 StgSharp.Common/Internal/Intrinsic/SIMDID.cs diff --git a/StgSharp.Common/Environment/Platform.cs b/StgSharp.Common/Environment/Platform.cs index e62b875d..0e854540 100644 --- a/StgSharp.Common/Environment/Platform.cs +++ b/StgSharp.Common/Environment/Platform.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="Platform" // Project: StgSharp @@ -55,15 +55,28 @@ namespace StgSharp private static uint markCount = 0; internal static GraphicAPI API = default; - public static int DefaultSIMDAlignment => NativeIntrinsic.IntrinsicLevel switch + public static unsafe int DefaultSIMDAlignment { - IntrinsicLevel.SSE => 16, - IntrinsicLevel.AVX2 => 32, - IntrinsicLevel.AVX512 => 64, - IntrinsicLevel.NEON => 16, - IntrinsicLevel.Non => 8, - _ => 8, - }; + get + { + SIMDID id = NativeIntrinsic.IntrinsicMask; + switch (id.MaskByte[0] & 0b_00001111) + { + case 1: + return id.MaskByte[1] switch + { + 1 => 16, + 2 => 32, + 3 => 64, + 4 => 64, + 5 => 64, + _ => 8, + }; + default: + return 8; + } + } + } public static ModuleToInitializeCollection Initialize() { @@ -71,31 +84,41 @@ namespace StgSharp } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void LogError(Exception ex) + public static void LogError( + Exception ex + ) { DefaultLog.InternalWriteLog(ex.Message, LogType.Error); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void LogInfo(string log) + public static void LogInfo( + string log + ) { DefaultLog.InternalWriteLog(log, LogType.Info); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void LogWarning(string log) + public static void LogWarning( + string log + ) { DefaultLog.InternalWriteLog(log, LogType.Warning); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void LogWarning(Exception notVerySeriesException) + public static void LogWarning( + Exception notVerySeriesException + ) { DefaultLog.InternalWriteLog(notVerySeriesException.Message, LogType.Warning); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Mark(bool needConsole) + public static void Mark( + bool needConsole + ) { markCount++; if (needConsole) { @@ -103,7 +126,9 @@ namespace StgSharp } } - public static void Terminate(int exitCode) + public static void Terminate( + int exitCode + ) { MainTimeProvider.StopProvidingTime(); GraphicFramework.glfwTerminate(); @@ -116,7 +141,8 @@ namespace StgSharp bool isConsole, [CallerFilePath] string filePath = "", [CallerMemberName] string callerName = "", - [CallerLineNumber] int lineNumber = 0) + [CallerLineNumber] int lineNumber = 0 + ) { Type t = target.GetType(); string log = $"Tracked object: {nameof(target)}\t\tType:{t.Name}\n"; @@ -147,12 +173,17 @@ namespace StgSharp /// The severity of current message. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void WriteLog(string message, LogType logType) + public static void WriteLog( + string message, + LogType logType + ) { DefaultLog.InternalWriteLog(message, logType); } - internal static TDele ConvertAPI(IntPtr funcPtr) + internal static TDele ConvertAPI( + IntPtr funcPtr + ) { if (funcPtr == IntPtr.Zero) { throw new InvalidCastException( @@ -168,7 +199,10 @@ namespace StgSharp } } - internal static Delegate ConvertAPI(IntPtr funcPtr, Type T) + internal static Delegate ConvertAPI( + IntPtr funcPtr, + Type T + ) { if (funcPtr == IntPtr.Zero) { throw new InvalidCastException( @@ -197,12 +231,13 @@ namespace StgSharp $"Program {Assembly.GetEntryAssembly()!.FullName} on {Environment.MachineName} Started.", LogType.Info); MainThreadID = System.Environment.CurrentManagedThreadId; - + /**/ + Dialogue.NeedShowWhenStartup = false; Dialogue.LoadDialogue(); if (Dialogue.NeedShowWhenStartup) { Dialogue.CreateDialogueProcessIfNotExist(); } - + /**/ NativeIntrinsic.InitIntrinsicContext(); } diff --git a/StgSharp.Common/HighPerformance/MatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs index 2c735db9..c99fd1dd 100644 --- a/StgSharp.Common/HighPerformance/MatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -46,13 +46,17 @@ namespace StgSharp.HighPerformance return (MatrixPanel*)NativeMemory.AlignedAlloc(Size(), AlignmentBytes); } - public static unsafe void Destroy(MatrixPanel* panel) where T : unmanaged, INumber + public static unsafe void Destroy( + MatrixPanel* panel + ) where T : unmanaged, INumber { NativeMemory.AlignedFree(panel); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe Span Elements(ref T head) where T : unmanaged, INumber + public static unsafe Span Elements( + ref T head + ) where T : unmanaged, INumber { void* p = Unsafe.AsPointer(ref head); return new Span(p, MaxElementCount()); @@ -92,17 +96,26 @@ namespace StgSharp.HighPerformance [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static int KernelSideCount() where T : unmanaged, INumber { - return NativeIntrinsic.IntrinsicLevel switch + SIMDID id = NativeIntrinsic.IntrinsicMask; + switch (id.MaskByte[0] & 0b_00001111) { - IntrinsicLevel.SSE => KernelSideCountSse(), - IntrinsicLevel.AVX2 => KernelSideCountAvx2(), - IntrinsicLevel.AVX512 => KernelSideCountAvx512(), - _ => 0 - }; + case 1: + return id.MaskByte[1] switch + { + 1 => KernelSideCountSse(), + 2 => KernelSideCountAvx2(), + 3 => KernelSideCountAvx512(), + _ => throw new NotSupportedException("Unsupported SIMD level."), + }; + default: + throw new NotSupportedException("Current hardware platform is not supported"); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe nuint Align64(void* p) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); + private static unsafe nuint Align64( + void* p + ) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int KernelSideCountAvx2() where T : unmanaged, INumber @@ -123,7 +136,9 @@ namespace StgSharp.HighPerformance } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int PanelSideFromBits(int vectorBits) where T : unmanaged, INumber + private static int PanelSideFromBits( + int vectorBits + ) where T : unmanaged, INumber { int elemBits = sizeof(T) * 8; @@ -137,13 +152,20 @@ namespace StgSharp.HighPerformance [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int PanelSideLength() where T : unmanaged, INumber { - return NativeIntrinsic.IntrinsicLevel switch + SIMDID id = NativeIntrinsic.IntrinsicMask; + switch (id.MaskByte[0] & 0b_00001111) { - IntrinsicLevel.SSE => PanelSideFromBits(128), - IntrinsicLevel.AVX2 => PanelSideFromBits(256), - IntrinsicLevel.AVX512 => PanelSideFromBits(512), - _ => 0 - }; + case 1: + return id.MaskByte[1] switch + { + 1 => PanelSideFromBits(128), + 2 => PanelSideFromBits(256), + 3 => PanelSideFromBits(512), + _ => throw new NotSupportedException("Unsupported SIMD level."), + }; + default: + throw new NotSupportedException("Current hardware platform is not supported"); + } } } @@ -154,7 +176,10 @@ namespace StgSharp.HighPerformance internal T Head; // first element of the panel buffer - public ref T this[int column, int row] + public ref T this[ + int column, + int row + ] { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => ref Unsafe.Add(ref Head, row * MatrixPanel.ElementSideLength() + column); diff --git a/StgSharp.Common/Internal/Intrinsic/SIMDID.cs b/StgSharp.Common/Internal/Intrinsic/SIMDID.cs new file mode 100644 index 00000000..7413a1a9 --- /dev/null +++ b/StgSharp.Common/Internal/Intrinsic/SIMDID.cs @@ -0,0 +1,51 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="SIMDID" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Internal.Intrinsic +{ + [StructLayout(LayoutKind.Explicit)] + internal unsafe struct SIMDID + { + + [FieldOffset(0)]public fixed byte MaskByte[16]; + [FieldOffset(0)]public ulong Mask; + + public SIMDID() + { + Unsafe.SkipInit(out this); + } + + } +} diff --git a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs index fa8ac761..a3db5f65 100644 --- a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs +++ b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs @@ -39,7 +39,7 @@ namespace StgSharp.Internal private static readonly IntrinsicContext _intrinsicContext = new(); - public static IntrinsicLevel IntrinsicLevel { get; set; } + public static SIMDID IntrinsicMask { get; set; } internal static IntrinsicContext Context { @@ -49,39 +49,32 @@ namespace StgSharp.Internal internal static void InitIntrinsicContext() { - int statCode; + ulong statCode = 0; fixed (IntrinsicContext* cPtr = &_intrinsicContext) { statCode = LoadIntrinsicFunction(cPtr); } - - switch (statCode) + SIMDID id = new() + { + Mask = statCode + }; + switch (id.MaskByte[1]) { case 0: - Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform")); Environment.Exit(-1); break; default: - IntrinsicLevel = (IntrinsicLevel)statCode; break; } + IntrinsicMask = id; } [LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] [UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])] - private static partial int LoadIntrinsicFunction(IntrinsicContext* context); - - } - - internal enum IntrinsicLevel : sbyte - { - - Non = 0, - SSE = 1, - AVX2 = 2, - AVX512 = 3, - NEON = -1 + private static partial ulong LoadIntrinsicFunction( + IntrinsicContext* context + ); } } \ No newline at end of file diff --git a/StgSharp.Common/Mathematics/Linear/Matrix.cs b/StgSharp.Common/Mathematics/Linear/Matrix.cs index c2527bec..5fbc9dbb 100644 --- a/StgSharp.Common/Mathematics/Linear/Matrix.cs +++ b/StgSharp.Common/Mathematics/Linear/Matrix.cs @@ -178,12 +178,9 @@ namespace StgSharp.Mathematics.Numeric ) { // Caller guarantees: indices in range, layout is dense square. - int k_c = y >> 2; - int k_r = x >> 2; - MatrixKernelEnumerator enumerator = new(_size.KernelColumnLength, _size.KernelRowLength, - k_c, k_r); - return Buffer + _boarder.GetIndexOffset(k_c, k_r, in enumerator); + x, y); + return Buffer + _boarder.GetIndexOffset(x, y, in enumerator); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/StgSharp.Common/Mathematics/Scalar/Random.cs b/StgSharp.Common/Mathematics/Scalar/Random.cs index b10d6107..d0451621 100644 --- a/StgSharp.Common/Mathematics/Scalar/Random.cs +++ b/StgSharp.Common/Mathematics/Scalar/Random.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="Random" // Project: StgSharp @@ -88,7 +88,7 @@ namespace StgSharp.Mathematics /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Random() where T: unmanaged, IConvertible, INumber + public static T Random() where T : unmanaged, IConvertible, INumber { if (randomSeed == null) @@ -97,13 +97,9 @@ namespace StgSharp.Mathematics string originSeed = ($"{((Assembly.GetEntryAssembly() == null) ? (Assembly.GetAssembly(typeof(World))!) : Assembly.GetEntryAssembly())}") + ($"{DateTime.UtcNow.ToString("o")}"); - using (SHA256 sha = SHA256.Create()) { - randomSeed = sha.ComputeHash(Encoding.UTF8.GetBytes(originSeed)); - } - } - using (SHA256 sha = SHA256.Create()) { - randomSeed = sha.ComputeHash(randomSeed); + randomSeed = SHA256.HashData(Encoding.UTF8.GetBytes(originSeed)); } + randomSeed = SHA256.HashData(randomSeed); fixed (byte* bptr = randomSeed) { return *(T*)bptr; } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs index 9e6f6731..e0083a1e 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs @@ -142,9 +142,13 @@ namespace StgSharp.Mathematics.Numeric { throw new IndexOutOfRangeException("Matrix dimensions must agree."); } + if (left.Layout != MatrixLayout.DenseRectangle || + right.Layout != MatrixLayout.DenseRectangle || + ans.Layout != MatrixLayout.DenseRectangle) { + throw new NotSupportedException("Only dense rectangle matrices are supported."); + } long count = (long)left.KernelColumnLength * left.KernelRowLength; - // int size = MatrixPanel.KernelSideCount(); if (count <= 1024) { MatrixParallelThread[] pool = null; @@ -155,15 +159,22 @@ namespace StgSharp.Mathematics.Numeric { for (int j = 0; j < ans.KernelColumnLength; j++) { - MatrixKernel* ansPanel = ans.GetKernelAddressUnsafe(i, j); - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); + MatrixKernel* ansPanel = GetKernelAddressUnsafe(ans.Buffer, + ans.KernelColumnLength, + i, j); - // Console.WriteLine($"{i}{j}"); + // NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); for (int k = 0; k < left.KernelColumnLength; k++) { NativeIntrinsic.Context - .mat[f32].kernel_fma((MatrixPanel*)left.GetKernelAddressUnsafe(i, k), - (MatrixPanel*)right.GetKernelAddressUnsafe(k, j), + .mat[f32].kernel_fma((MatrixPanel*)GetKernelAddressUnsafe(left.Buffer, + left.KernelColumnLength, + i, + k), + (MatrixPanel*)GetKernelAddressUnsafe(right.Buffer, + right.KernelColumnLength, + k, + j), (MatrixPanel*)ansPanel); } } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.cs index 27fff0db..902046e8 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.cs @@ -45,7 +45,6 @@ namespace StgSharp.Mathematics.Numeric ) { int f32 = (int)MatrixElementType.F32; - int size = MatrixPanel.KernelSideCount(); MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; @@ -62,17 +61,15 @@ namespace StgSharp.Mathematics.Numeric { int i = (int)(c / ansWidth); int j = (int)(c % ansWidth); - MatrixKernel* ansPanel = GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, - ansHeight, i, j); - NativeIntrinsic.Context.mat[f32].clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < commonK; k += size) { + for (int k = 0; k < commonK; k++) { NativeIntrinsic.Context .mat[f32].kernel_fma( (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)leftBuffer, ansHeight, k, j), (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)rightBuffer, commonK, j, k), - (MatrixPanel*)ansPanel); + (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, + ansHeight, i, j)); } } } diff --git a/StgSharp.Native/sn-src/cpuid.x86.c b/StgSharp.Native/sn-src/cpuid.x86.c index 85156cc4..23d6dfc8 100644 --- a/StgSharp.Native/sn-src/cpuid.x86.c +++ b/StgSharp.Native/sn-src/cpuid.x86.c @@ -78,7 +78,7 @@ static int is_intel_native_avx512(int family, int model) SN_API SIMDID SN_DECL sn_get_simd_level() { - if (!global_simdid.mask) { + if (global_simdid.mask) { return global_simdid; } diff --git a/StgSharp.Native/sn-src/m.fma.x86.c b/StgSharp.Native/sn-src/m.fma.x86.c index a620bb52..b0fc9f20 100644 --- a/StgSharp.Native/sn-src/m.fma.x86.c +++ b/StgSharp.Native/sn-src/m.fma.x86.c @@ -7,8 +7,8 @@ #define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) -// ans = left * right + ans -// right is not transposed +// float + DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) { const __m128 a0 = _mm_load_ps((float *)&left->f32_x[0]); @@ -57,56 +57,18 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma) ans->f32_x[3] = c3; } -DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) -{ - register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); - register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); - - register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); - register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); - - register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); - register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); - - //group 0 - register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0)); - register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0)); - c01 = _mm256_fmadd_ps(a01, b01_perm, c01); - c23 = _mm256_fmadd_ps(a23, b23_perm, c23); - - //group 2 - b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2)); - b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2)); - c01 = _mm256_fmadd_ps(a23, b01_perm, c01); - c23 = _mm256_fmadd_ps(a01, b23_perm, c23); - - //group 1 - register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21); - register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30); - b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1)); - b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1)); - c01 = _mm256_fmadd_ps(a12, b01_perm, c01); - c23 = _mm256_fmadd_ps(a03, b23_perm, c23); - - //group 3 - a12 = _mm256_permute2f128_ps(a01, a23, 0x21); - a03 = _mm256_permute2f128_ps(a01, a23, 0x30); - b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3)); - b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3)); - c01 = _mm256_fmadd_ps(a03, b01_perm, c01); - c23 = _mm256_fmadd_ps(a12, b23_perm, c23); - - ans->f32_y[0] = c01; - ans->f32_y[1] = c23; -} - DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) { + static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 }; + static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 }; + register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); + b01 = _mm256_permutevar_ps(b01, b01_shift); + b23 = _mm256_permutevar_ps(b23, b23_shift); register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); @@ -125,49 +87,186 @@ DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma) //group 1 register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21); - register __m256 a03 = _mm256_permute2f128_ps(a01, a23, 0x30); + register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30); b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1)); b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1)); c01 = _mm256_add_ps(_mm256_mul_ps(a12, b01_perm), c01); - c23 = _mm256_add_ps(_mm256_mul_ps(a03, b23_perm), c23); + c23 = _mm256_add_ps(_mm256_mul_ps(a30, b23_perm), c23); //group 3 - a12 = _mm256_permute2f128_ps(a01, a23, 0x21); - a03 = _mm256_permute2f128_ps(a01, a23, 0x30); b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3)); b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3)); - c01 = _mm256_add_ps(_mm256_mul_ps(a03, b01_perm), c01); + c01 = _mm256_add_ps(_mm256_mul_ps(a30, b01_perm), c01); c23 = _mm256_add_ps(_mm256_mul_ps(a12, b23_perm), c23); ans->f32_y[0] = c01; ans->f32_y[1] = c23; } +DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma) +{ + static const __m256i b01_shift = { 0, 1, 2, 3, 5, 6, 7, 4 }; + static const __m256i b23_shift = { 2, 3, 0, 1, 7, 4, 5, 6 }; + + register __m256 a01 = _mm256_load_ps((float *)&left->f32_y[0]); + register __m256 a23 = _mm256_load_ps((float *)&left->f32_y[1]); + + register __m256 b01 = _mm256_load_ps((float *)&right->f32_y[0]); + register __m256 b23 = _mm256_load_ps((float *)&right->f32_y[1]); + b01 = _mm256_permutevar_ps(b01, b01_shift); + b23 = _mm256_permutevar_ps(b23, b23_shift); + + register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); + register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); + + //group 0 + register __m256 b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(0)); + register __m256 b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(0)); + c01 = _mm256_fmadd_ps(a01, b01_perm, c01); + c23 = _mm256_fmadd_ps(a23, b23_perm, c23); + + //group 2 + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(2)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(2)); + c01 = _mm256_fmadd_ps(a23, b01_perm, c01); + c23 = _mm256_fmadd_ps(a01, b23_perm, c23); + + //group 1 + register __m256 a12 = _mm256_permute2f128_ps(a01, a23, 0x21); + register __m256 a30 = _mm256_permute2f128_ps(a01, a23, 0x30); + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(1)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(1)); + c01 = _mm256_fmadd_ps(a12, b01_perm, c01); + c23 = _mm256_fmadd_ps(a30, b23_perm, c23); + + //group 3 + b01_perm = _mm256_shuffle_ps(b01, b01, _FMA_SELECT(3)); + b23_perm = _mm256_shuffle_ps(b23, b23, _FMA_SELECT(3)); + c01 = _mm256_fmadd_ps(a30, b01_perm, c01); + c23 = _mm256_fmadd_ps(a12, b23_perm, c23); + + ans->f32_y[0] = c01; + ans->f32_y[1] = c23; +} + DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma) { + static const __m512i b_shift = { 0, 1, 2, 3, 5, 6, 7, 4, 10, 11, 8, 9, 15, 12, 13, 14 }; + register __m512 a = _mm512_load_ps((float *)left); register __m512 b = _mm512_load_ps((float *)right); register __m512 c = _mm512_load_ps((float *)ans); + b = _mm512_permutexvar_ps(b_shift, b); register __m512 a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 1, 0)); register __m512 b_perm = _mm512_permute_ps(b, _FMA_SELECT(0)); - c = _mm512_fmadd_ps(a, b_perm, c); + c = _mm512_fmadd_ps(a_perm, b_perm, c); a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(2, 1, 0, 3)); b_perm = _mm512_permute_ps(b, _FMA_SELECT(1)); - c = _mm512_fmadd_ps(a, b_perm, c); + c = _mm512_fmadd_ps(a_perm, b_perm, c); a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(1, 0, 3, 2)); b_perm = _mm512_permute_ps(b, _FMA_SELECT(2)); - c = _mm512_fmadd_ps(a, b_perm, c); + c = _mm512_fmadd_ps(a_perm, b_perm, c); a_perm = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(0, 3, 2, 1)); b_perm = _mm512_permute_ps(b, _FMA_SELECT(3)); - c = _mm512_fmadd_ps(a, b_perm, c); + c = _mm512_fmadd_ps(a_perm, b_perm, c); _mm512_store_ps((float *)ans, c); } +// double + +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma) +{ + const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]); + + __m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]); + __m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]); + __m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]); + __m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]); + + register __m256d ccol; + + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[0]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[1]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[2]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[3]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; +} + +DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma) +{ + const __m256d a0 = _mm256_load_pd((double *)&left->f64_y[0]); + const __m256d a1 = _mm256_load_pd((double *)&left->f64_y[1]); + const __m256d a2 = _mm256_load_pd((double *)&left->f64_y[2]); + const __m256d a3 = _mm256_load_pd((double *)&left->f64_y[3]); + + __m256d b0 = _mm256_load_pd((double *)&right->f64_y[0]); + __m256d b1 = _mm256_load_pd((double *)&right->f64_y[1]); + __m256d b2 = _mm256_load_pd((double *)&right->f64_y[2]); + __m256d b3 = _mm256_load_pd((double *)&right->f64_y[3]); + + register __m256d ccol; + + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[0]); + ccol = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(b0, _FMA_SELECT(0)), ccol); + ccol = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(b0, _FMA_SELECT(1)), ccol); + ccol = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(b0, _FMA_SELECT(2)), ccol); + ccol = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(b0, _FMA_SELECT(3)), ccol); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[1]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b1, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b1, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b1, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b1, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[2]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b2, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b2, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b2, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b2, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; + /* Column 0 */ + ccol = _mm256_load_pd((double *)&ans->f32_y[3]); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a0, _mm256_permute4x64_pd(b3, _FMA_SELECT(0)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a1, _mm256_permute4x64_pd(b3, _FMA_SELECT(1)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a2, _mm256_permute4x64_pd(b3, _FMA_SELECT(2)))); + ccol = _mm256_add_pd(ccol, _mm256_mul_pd(a3, _mm256_permute4x64_pd(b3, _FMA_SELECT(3)))); + ans->f64_y[0] = ccol; +} + DECLARE_PNL_PROC_LEFT_RIGHT_ANS(double, sse, , fma) { for (int col = 0; col < 4; ++col) { diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 9f67c56f..8878d99a 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -66,19 +66,13 @@ namespace StgSharp hlsf); MatrixCompute.Fill(p, 0.0f); - /* - for (int i = 0; i < 10; i++) + /**/ + for (int j = 0; j < 20; j++) { - MatrixCompute.Add(n, m, n); - for (int j = 0; j < 5; j++) - { - for (int k = 0; k < 5; k++) { - Console.Write($"{n[j, k]:0.0E+00} \t"); - } - Console.WriteLine(); + for (int k = 0; k < 10; k++) { + Console.Write($"{m[j, k]:0.0E+00}\t"); } - Console.WriteLine($"{i}"); - GC.Collect(); + Console.WriteLine(); } /**/ @@ -86,8 +80,8 @@ namespace StgSharp MatrixCompute.Mul(n, m, p); for (int i = 0; i < 20; i++) { - for (int j = 0; j < 20; j++) { - Console.Write($"{p[i, j]:0.0E+00} \t"); + for (int j = 0; j < 10; j++) { + Console.Write($"{p[i, j]:0.0E+00}\t"); } Console.WriteLine(); }