From 631bc2b6e89cfc732773fc075abbe0084837e980 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:52:17 +0800 Subject: [PATCH 1/4] working on matrix mul matrix --- .../HighPerformance/MatrixKernel.cs | 11 +- .../HighPerformance/MatrixPanel.cs | 270 +++++------------- .../Internal/Intrinsic/IntrinsicContext.cs | 4 +- .../Intrinsic/intrinsicContextProvider.cs | 2 +- .../Numeric/MatrixCompute.Basic.cs | 33 ++- .../Numeric/TaskPackageUsage.md | 0 StgSharp.Native/sn-src/k_fma.c | 43 +++ cache.txt | 9 - 8 files changed, 151 insertions(+), 221 deletions(-) delete mode 100644 StgSharp.Mathematics/Numeric/TaskPackageUsage.md diff --git a/StgSharp.Common/HighPerformance/MatrixKernel.cs b/StgSharp.Common/HighPerformance/MatrixKernel.cs index a460dd1a..4535487f 100644 --- a/StgSharp.Common/HighPerformance/MatrixKernel.cs +++ b/StgSharp.Common/HighPerformance/MatrixKernel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixKernel" // Project: StgSharp @@ -32,9 +32,16 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { + public unsafe struct MatrixKernel + { + + private byte Head; + + } + [StructLayout(LayoutKind.Sequential)] [InlineArray(16)] - public unsafe struct MatrixKernel where T: unmanaged, INumber + public unsafe struct MatrixKernel where T : unmanaged, INumber { private T c0r0; diff --git a/StgSharp.Common/HighPerformance/MatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs index c400c8c9..2e09dff3 100644 --- a/StgSharp.Common/HighPerformance/MatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixPanel" // Project: StgSharp @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Internal; using System; using System.Numerics; using System.Runtime.CompilerServices; @@ -32,267 +33,130 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { - #pragma warning disable CA1000 - - public interface IMatrixPanel : IDisposable where T : unmanaged,INumber + public unsafe struct MatrixPanel { - ref T this[int column, int row] { get; } - - static abstract int MaxElementCount { get; } - - static abstract int ElementSideLength { get; } - - static abstract nuint Size { get; } - - void Dispose(); + private byte Head; } - /// - /// Buffer for SSE targeted matrix panel, size of a panel is 4x4 elements, and no smaller than - /// 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixPanelSSE : IMatrixPanel where T : unmanaged, INumber + // Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct. + public unsafe struct MatrixPanel where T : unmanaged, INumber { - private T element; + public const int AlignmentBytes = 64; - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixPanelSSE() { } + internal T Head; // first element of the panel buffer public ref T this[int column, int row] { - get - { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => ref Unsafe.Add(ref Head, row * ElementSideLength + column); } public static int ElementSideLength { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(4, 16 / sizeof(T)); + get => Math.Max(8, AlignmentBytes / sizeof(T)); + } public static int MaxElementCount { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixPanelSSE* Create() - { - MatrixPanelSSE* kernel = (MatrixPanelSSE*)NativeMemory.AlignedAlloc(Size, 16); - return kernel; + get => ElementSideLength * ElementSideLength; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixPanelSSE* kernel) + public static nuint Size { - NativeMemory.AlignedFree(kernel); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (nuint)(MaxElementCount * sizeof(T)); } - public void Dispose() - { - fixed (MatrixPanelSSE* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M128 GetVectorChunk(int column, int chunk) - { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); - } - - // SIMD access helpers (column-major) - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 16 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M128 value) - { - GetVectorChunk(column, chunk) = value; - } - - } - - /// - /// Buffer for AVX targeted matrix panel, size of a panel is 4x4 elements, and no smaller than - /// 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixPanelAVX : IMatrixPanel where T : unmanaged, INumber - { - - private T element; - - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixPanelAVX() { } - - public ref T this[int column, int row] + public Span Elements { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); + ref T first = ref Head; + unsafe + { + void* p = Unsafe.AsPointer(ref first); + return new Span(p, MaxElementCount); + } } } - public static int ElementSideLength + // Allocate aligned memory for one panel; caller must free via Destroy. + public static unsafe MatrixPanel* Create() { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(8, 32 / sizeof(T)); - - public static unsafe int MaxElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixPanelAVX* Create() - { - MatrixPanelAVX* kernel = (MatrixPanelAVX*)NativeMemory.AlignedAlloc(Size, 32); - return kernel; + return (MatrixPanel*)NativeMemory.AlignedAlloc(Size, AlignmentBytes); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixPanelAVX* kernel) + public static unsafe void Destroy(MatrixPanel* panel) { - NativeMemory.AlignedFree(kernel); + NativeMemory.AlignedFree(panel); } - public void Dispose() + // View an existing pointer; caller guarantees lifetime/alignment. + public static unsafe MatrixPanel* FromPointer(void* ptr) { - fixed (MatrixPanelAVX* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } + return (MatrixPanel*)Align64(ptr); } + /// + /// Number of 4x4 kernels that fit in a panel for the given ISA and element T. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M256 GetVectorChunk(int column, int chunk) + internal static int KernelCapacity() { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + int perSide = KernelSideCount(); + return perSide * perSide; } - // SIMD access helpers for AVX (column-major) + /// + /// How many 4x4 kernels fit along a single panel dimension for the given ISA. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 32 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M256 value) + internal static int KernelSideCount() { - GetVectorChunk(column, chunk) = value; - } - - } - - /// - /// Buffer for AVX512 targeted matrix panel, size of a panel is 4x4 elements, and no smaller - /// than 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixComputeKernelAVX512 : IMatrixPanel - where T : unmanaged, INumber - { - - private T element; - - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixComputeKernelAVX512() { } - - public ref T this[int column, int row] - { - get + return NativeIntrinsic.IntrinsicLevel switch { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); - } - } - - public static int ElementSideLength - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(8, 64 / sizeof(T)); - - public static unsafe int MaxElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixComputeKernelAVX512* Create() - { - MatrixComputeKernelAVX512* kernel = (MatrixComputeKernelAVX512*)NativeMemory.AlignedAlloc(Size, 64); - return kernel; + IntrinsicLevel.SSE => KernelSideCountSse(), + IntrinsicLevel.AVX2 => KernelSideCountAvx2(), + IntrinsicLevel.AVX512 => KernelSideCountAvx512(), + _ => 0 + }; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixComputeKernelAVX512* kernel) - { - NativeMemory.AlignedFree(kernel); - } + private static unsafe nuint Align64(void* p) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); - public void Dispose() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int KernelSideCountAvx2() { - fixed (MatrixComputeKernelAVX512* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } + const int minQ = 8; + int lanes = 32 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M512 GetVectorChunk(int column, int chunk) + private static int KernelSideCountAvx512() { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + const int minQ = 8; + int lanes = 64 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } - // SIMD access helpers for AVX-512 (column-major) [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 64 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M512 value) + private static int KernelSideCountSse() { - GetVectorChunk(column, chunk) = value; + const int minQ = 4; + int lanes = 16 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } -#pragma warning restore CA1000 + } -} \ No newline at end of file +} diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index e3830279..53c1ffdb 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="IntrinsicContext" // Project: StgSharp @@ -44,7 +44,7 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] f32_buffer_scalar_mul; //f32_buffer_s_m public delegate* unmanaged[Cdecl] f32_buffer_sub; //f32_kernel_sub public delegate* unmanaged[Cdecl] f32_buffer_transpose; //f32_kernel_transpose - public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel + public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel public delegate* unmanaged[Cdecl] f32_normalize_3; //f32_normalize_3 public delegate* unmanaged[Cdecl] f32_panel_fma; //f32_panel_fma public delegate* unmanaged[Cdecl] f32_store_panel; //f32_store_panel diff --git a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs index 2fabc0a3..eaf92a40 100644 --- a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs +++ b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs @@ -59,7 +59,7 @@ namespace StgSharp.Internal { case 0: - // Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform")); + Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform")); Environment.Exit(-1); break; default: diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index 52cec027..a0c05ae5 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixCompute.Basic" // Project: StgSharp @@ -30,6 +30,7 @@ using StgSharp.HighPerformance; using StgSharp.Internal; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; using System.Linq; using System.Numerics; using System.Security.AccessControl; @@ -124,11 +125,35 @@ namespace StgSharp.Mathematics.Numeric Matrix right, Matrix ans) { - if (left.RowLength != right.ColumnLength || - left.RowLength != ans.RowLength || - right.ColumnLength != ans.ColumnLength) { + // For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN + // ColumnLength = number of columns (N), RowLength = number of rows (M) + if (left.ColumnLength != right.RowLength || + ans.ColumnLength != right.ColumnLength || + ans.RowLength != left.RowLength) + { throw new IndexOutOfRangeException("Matrix dimensions must agree."); } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + if (count <= 256) + { + MatrixPanel* leftPanel = MatrixPanel.Create(); + MatrixPanel* rightPanel = MatrixPanel.Create(); + MatrixPanel* ansPanel = MatrixPanel.Create(); + int size = MatrixPanel.KernelSideCount(); + for (int i = 0; i < left.RowLength; i += size) + { + for (int j = 0; j < right.ColumnLength; j += size) + { + for (int k = 0; k < left.ColumnLength; k += size) { + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixKernel*)left.Buffer, + (MatrixPanel*)leftPanel, + left.ColumnLength, left.RowLength, i, + k); + } + } + } + } throw new NotImplementedException(); } diff --git a/StgSharp.Mathematics/Numeric/TaskPackageUsage.md b/StgSharp.Mathematics/Numeric/TaskPackageUsage.md deleted file mode 100644 index e69de29b..00000000 diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index d5e8510c..11ad5d05 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -52,6 +52,49 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma) PANEL_COL_VEC((*ans), 3, 0) = c3; } +// ans = left * right + ans +// right is not transposed +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma) +{ + static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 }; + + for (int col = 0; col < 8; ++col) { + register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]); + register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]); + register __m256i idx = _mm256_setzero_si256(); + register __m256i increament = _mm256_load_si256(&idx0); + + register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]); + register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]); + register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]); + register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]); + + ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + + a0 = _mm256_load_ps((float *)&left->vec[4][0]); + a1 = _mm256_load_ps((float *)&left->vec[5][0]); + a2 = _mm256_load_ps((float *)&left->vec[6][0]); + a3 = _mm256_load_ps((float *)&left->vec[7][0]); + + ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); + + _mm256_store_ps((float *)&ans->vec[col][0], ccol); + } +} + // ans = left * right + ans // right is not transposed DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma) diff --git a/cache.txt b/cache.txt index c5e7fe66..e69de29b 100644 --- a/cache.txt +++ b/cache.txt @@ -1,9 +0,0 @@ -public const uint GL_ACTIVE_SUBROUTINES = 0x8DE5 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORMS = 0x8DE6 -public const uint GL_MAX_SUBROUTINES = 0x8DE7 -public const uint GL_MAX_SUBROUTINE_UNIFORM_LOCATIONS = 0x8DE8 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORM_LOCATIONS = 0x8E47 -public const uint GL_ACTIVE_SUBROUTINE_MAX_LENGTH = 0x8E48 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORM_MAX_LENGTH = 0x8E49 -public const uint GL_NUM_COMPATIBLE_SUBROUTINES = 0x8E4A -public const uint GL_COMPATIBLE_SUBROUTINES = 0x8E4B \ No newline at end of file From 9ff438d3b96db9297d0e222b4631fcae5c7d4e7d Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 11 Dec 2025 18:09:35 +0800 Subject: [PATCH 2/4] working on matrix mul matrix --- .../HighPerformance/MatrixPanel.cs | 122 +++++++++--------- .../Internal/Intrinsic/IntrinsicContext.cs | 1 + .../Numeric/MatrixCompute.Additional.cs | 54 ++++++++ .../Numeric/MatrixCompute.Basic.cs | 93 ++++++++----- StgSharp.Mathematics/Numeric/MatrixCompute.cs | 37 ++++++ .../Numeric/MatrixComputeModel.cs | 15 ++- .../Numeric/MatrixParallel.cs | 13 +- .../Numeric/MatrixParallelThread.cs | 14 +- StgSharp.Native/sn-head/sn_intrinsic.h | 11 +- StgSharp.Native/sn-head/sn_matkernel.h | 4 + StgSharp.Native/sn-src/k_basicop.c | 27 ++++ StgSharp.Native/sn-src/k_panel.c | 91 +++++++++++++ StgSharp.Native/sn-src/load_intrinsic.c | 12 +- 13 files changed, 379 insertions(+), 115 deletions(-) create mode 100644 StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs create mode 100644 StgSharp.Mathematics/Numeric/MatrixCompute.cs diff --git a/StgSharp.Common/HighPerformance/MatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs index 2e09dff3..7f3ae6d0 100644 --- a/StgSharp.Common/HighPerformance/MatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -34,82 +34,63 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { public unsafe struct MatrixPanel - { - - private byte Head; - - } - - // Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct. - public unsafe struct MatrixPanel where T : unmanaged, INumber { public const int AlignmentBytes = 64; - internal T Head; // first element of the panel buffer - - public ref T this[int column, int row] - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => ref Unsafe.Add(ref Head, row * ElementSideLength + column); - } - - public static int ElementSideLength - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => Math.Max(8, AlignmentBytes / sizeof(T)); - } - - public static int MaxElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => ElementSideLength * ElementSideLength; - } - - public static nuint Size - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => (nuint)(MaxElementCount * sizeof(T)); - } - - public Span Elements - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get - { - ref T first = ref Head; - unsafe - { - void* p = Unsafe.AsPointer(ref first); - return new Span(p, MaxElementCount); - } - } - } + private byte Head; // Allocate aligned memory for one panel; caller must free via Destroy. - public static unsafe MatrixPanel* Create() + public static unsafe MatrixPanel* Create() where T : unmanaged, INumber { - return (MatrixPanel*)NativeMemory.AlignedAlloc(Size, AlignmentBytes); + return (MatrixPanel*)NativeMemory.AlignedAlloc(Size(), AlignmentBytes); } - public static unsafe void Destroy(MatrixPanel* panel) + public static unsafe void Destroy(MatrixPanel* panel) where T : unmanaged, INumber { NativeMemory.AlignedFree(panel); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe Span Elements(ref T head) where T : unmanaged, INumber + { + void* p = Unsafe.AsPointer(ref head); + return new Span(p, MaxElementCount()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int ElementSideLength() where T : unmanaged, INumber + { + return Math.Max(8, AlignmentBytes / sizeof(T)); + } + // View an existing pointer; caller guarantees lifetime/alignment. - public static unsafe MatrixPanel* FromPointer(void* ptr) + public static unsafe MatrixPanel* FromPointer(void* ptr) + where T : unmanaged, INumber { return (MatrixPanel*)Align64(ptr); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int MaxElementCount() where T : unmanaged, INumber + { + int side = ElementSideLength(); + return side * side; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static nuint Size() where T : unmanaged, INumber + { + return (nuint)(MaxElementCount() * sizeof(T)); + } + /// /// Number of 4x4 kernels that fit in a panel for the given ISA and element T. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int KernelCapacity() + internal static int KernelCapacity() where T : unmanaged, INumber { - int perSide = KernelSideCount(); + int perSide = KernelSideCount(); return perSide * perSide; } @@ -117,13 +98,13 @@ namespace StgSharp.HighPerformance /// How many 4x4 kernels fit along a single panel dimension for the given ISA. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int KernelSideCount() + internal static int KernelSideCount() where T : unmanaged, INumber { return NativeIntrinsic.IntrinsicLevel switch { - IntrinsicLevel.SSE => KernelSideCountSse(), - IntrinsicLevel.AVX2 => KernelSideCountAvx2(), - IntrinsicLevel.AVX512 => KernelSideCountAvx512(), + IntrinsicLevel.SSE => KernelSideCountSse(), + IntrinsicLevel.AVX2 => KernelSideCountAvx2(), + IntrinsicLevel.AVX512 => KernelSideCountAvx512(), _ => 0 }; } @@ -132,7 +113,7 @@ namespace StgSharp.HighPerformance private static unsafe nuint Align64(void* p) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int KernelSideCountAvx2() + private static int KernelSideCountAvx2() where T : unmanaged, INumber { const int minQ = 8; int lanes = 32 / sizeof(T); @@ -141,7 +122,7 @@ namespace StgSharp.HighPerformance } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int KernelSideCountAvx512() + private static int KernelSideCountAvx512() where T : unmanaged, INumber { const int minQ = 8; int lanes = 64 / sizeof(T); @@ -150,7 +131,7 @@ namespace StgSharp.HighPerformance } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int KernelSideCountSse() + private static int KernelSideCountSse() where T : unmanaged, INumber { const int minQ = 4; int lanes = 16 / sizeof(T); @@ -159,4 +140,25 @@ namespace StgSharp.HighPerformance } } + + // Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct. + public unsafe struct MatrixPanel where T : unmanaged, INumber + { + + internal T Head; // first element of the panel buffer + + public ref T this[int column, int row] + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => ref Unsafe.Add(ref Head, row * MatrixPanel.ElementSideLength() + column); + } + + public Span Elements + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => MatrixPanel.Elements(ref Head); + } + + } } + diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index 53c1ffdb..064daceb 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -45,6 +45,7 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] f32_buffer_sub; //f32_kernel_sub public delegate* unmanaged[Cdecl] f32_buffer_transpose; //f32_kernel_transpose public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel + public delegate* unmanaged[Cdecl] f32_clear_panel; public delegate* unmanaged[Cdecl] f32_normalize_3; //f32_normalize_3 public delegate* unmanaged[Cdecl] f32_panel_fma; //f32_panel_fma public delegate* unmanaged[Cdecl] f32_store_panel; //f32_store_panel diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs new file mode 100644 index 00000000..9b3b2030 --- /dev/null +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs @@ -0,0 +1,54 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixCompute.Additional" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.HighPerformance; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public partial class MatrixCompute + { + + private static unsafe void ComputeFmaPanelF32( + MatrixKernel* left, + MatrixKernel* right, + MatrixKernel* ans, + ScalarPacket* scalar) + { + const int eSize = 32; + int colum = scalar->Data(0); + int row = scalar->Data(1); + int k = + + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index a0c05ae5..99bfe321 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -39,13 +39,10 @@ using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public static class MatrixCompute + public static partial class MatrixCompute { - public static unsafe Matrix Add( - Matrix left, - Matrix right, - Matrix ans) + public static unsafe void Add(Matrix left, Matrix right, Matrix ans) { if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength || @@ -61,7 +58,7 @@ namespace StgSharp.Mathematics.Numeric MatrixKernel* ansPtr = ans.Buffer; NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); - return ans; + return; } else if (count <= 1024) { MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); @@ -72,22 +69,22 @@ namespace StgSharp.Mathematics.Numeric NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); MatrixParallel.ReturnParallelResources(ref pool); - return ans; + return; } MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Result = (nint)ans.Buffer; package->ElementSize = sizeof(float); - package->PrimCount = left.KernelColumnLength; - package->SecCount = left.KernelRowLength; + package->PrimCount = ans.KernelColumnLength; + package->SecCount = ans.KernelRowLength; package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_add; MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); taskGroup.Dispose(); - return ans; + return; } public static unsafe void Fill(Matrix ans, float value) @@ -120,10 +117,7 @@ namespace StgSharp.Mathematics.Numeric taskGroup.Dispose(); } - public static unsafe Matrix Mul( - Matrix left, - Matrix right, - Matrix ans) + public static unsafe void Mul(Matrix left, Matrix right, Matrix ans) { // For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN // ColumnLength = number of columns (N), RowLength = number of rows (M) @@ -134,30 +128,68 @@ namespace StgSharp.Mathematics.Numeric throw new IndexOutOfRangeException("Matrix dimensions must agree."); } long count = (long)left.KernelColumnLength * left.KernelRowLength; - if (count <= 256) + if (count <= 1024) { - MatrixPanel* leftPanel = MatrixPanel.Create(); - MatrixPanel* rightPanel = MatrixPanel.Create(); - MatrixPanel* ansPanel = MatrixPanel.Create(); - int size = MatrixPanel.KernelSideCount(); + MatrixParallelThread[] pool = null; + if (count > 256) { + pool = MatrixParallel.LeadParallel(1); + } + MatrixPanel* leftPanel = MatrixPanel.Create(); + MatrixPanel* rightPanel = MatrixPanel.Create(); + MatrixPanel* ansPanel = MatrixPanel.Create(); + int size = MatrixPanel.KernelSideCount(); for (int i = 0; i < left.RowLength; i += size) { for (int j = 0; j < right.ColumnLength; j += size) { - for (int k = 0; k < left.ColumnLength; k += size) { + NativeIntrinsic.Intrinsic.f32_clear_panel((MatrixPanel*)ansPanel); + for (int k = 0; k < left.ColumnLength; k += size) + { NativeIntrinsic.Intrinsic .f32_build_panel((MatrixKernel*)left.Buffer, (MatrixPanel*)leftPanel, left.ColumnLength, left.RowLength, i, k); + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixKernel*)right.Buffer, + (MatrixPanel*)rightPanel, + right.ColumnLength, right.RowLength, k, + j); + NativeIntrinsic.Intrinsic + .f32_panel_fma((MatrixPanel*)leftPanel, + (MatrixPanel*)rightPanel, + (MatrixPanel*)ansPanel); } + NativeIntrinsic.Intrinsic + .f32_store_panel((MatrixKernel*)ans.Buffer, + (MatrixPanel*)ansPanel, ans.ColumnLength, + ans.RowLength, i, j); } } + MatrixPanel.Destroy(leftPanel); + MatrixPanel.Destroy(rightPanel); + MatrixPanel.Destroy(ansPanel); + if (count > 256) { + MatrixParallel.ReturnParallelResources(ref pool!); + } + return; } - throw new NotImplementedException(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Right = (nint)left.Buffer; + package->Result = (nint)ans.Buffer; + package->ElementSize = sizeof(float); + package->PrimCount = left.KernelColumnLength; + package->SecCount = left.KernelRowLength; + package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); + package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul; + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.Dispose(); + return; } - public static unsafe Matrix Mul(Matrix left, float right, Matrix ans) + public static unsafe void Mul(Matrix left, float right, Matrix ans) { if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { throw new IndexOutOfRangeException("Matrix dimensions must agree."); @@ -171,7 +203,7 @@ namespace StgSharp.Mathematics.Numeric MatrixKernel* ansPtr = ans.Buffer; NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); - return ans; + return; } else if (count <= 1024) { MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); @@ -181,7 +213,7 @@ namespace StgSharp.Mathematics.Numeric NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); MatrixParallel.ReturnParallelResources(ref pool); - return ans; + return; } MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Right = (nint)left.Buffer; @@ -197,13 +229,10 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); taskGroup.Dispose(); MatrixParallelFactory.Release(scalar); - return ans; + return; } - public static unsafe Matrix Sub( - Matrix left, - Matrix right, - Matrix ans) + public static unsafe void Sub(Matrix left, Matrix right, Matrix ans) { if (left.ColumnLength != right.ColumnLength || left.RowLength != right.RowLength || @@ -219,7 +248,7 @@ namespace StgSharp.Mathematics.Numeric MatrixKernel* ansPtr = ans.Buffer; NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); - return ans; + return; } else if (count <= 1024) { MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); @@ -230,7 +259,7 @@ namespace StgSharp.Mathematics.Numeric NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); MatrixParallel.ReturnParallelResources(ref pool); - return ans; + return; } MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; @@ -245,7 +274,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); taskGroup.Dispose(); - return ans; + return; } } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.cs new file mode 100644 index 00000000..829d6fc1 --- /dev/null +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.cs @@ -0,0 +1,37 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixCompute" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public static partial class MatrixCompute { } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs index f0012e5f..a96db5e0 100644 --- a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixComputeModel" // Project: StgSharp @@ -76,12 +76,13 @@ namespace StgSharp.Mathematics.Numeric long offset = Unsafe.As(ref p->PrimTileOffset) * size; ((delegate* unmanaged[Cdecl]< void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Right + offset), (void*)(p->Result + offset), - p->Scalar, count); + p->Scalar, + count); MatrixParallelFactory.Release(p); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute_Binary(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeBinary(MatrixParallelTaskPackage* p) { int eSize = p->ElementSize; @@ -133,7 +134,7 @@ namespace StgSharp.Mathematics.Numeric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute_BinaryScalar(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeBinaryScalar(MatrixParallelTaskPackage* p) { int eSize = p->ElementSize; @@ -185,7 +186,7 @@ namespace StgSharp.Mathematics.Numeric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute_Unary(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeUnary(MatrixParallelTaskPackage* p) { int eSize = p->ElementSize; @@ -230,7 +231,7 @@ namespace StgSharp.Mathematics.Numeric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute_UnaryScalar(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeUnaryScalar(MatrixParallelTaskPackage* p) { int eSize = p->ElementSize; @@ -274,5 +275,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelFactory.Release(p); } + internal static unsafe void PanelComputeBinary(MatrixParallelTaskPackage* p) { } + } } \ No newline at end of file diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.cs index ecf98cb2..b2619463 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixParallel" // Project: StgSharp @@ -27,15 +27,14 @@ // ----------------------------------------------------------------------- using StgSharp.Collections; using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using System.Threading; +using System.Linq; +using System.Text; +using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public static partial class MatrixParallel + public static class MatrixParallel { private static bool _meaningful; @@ -97,4 +96,4 @@ namespace StgSharp.Mathematics.Numeric } } -} \ No newline at end of file +} diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index d2afce4b..0afa39ef 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixParallelThread" // Project: StgSharp @@ -114,16 +114,16 @@ namespace StgSharp.Mathematics.Numeric switch (p->ComputeMode.ParameterStyle) { case MatrixOperationParam.RIGHT_ANS_PARAM: - MatrixComputeModel.Compute_Unary(p); + MatrixComputeModel.KernelComputeUnary(p); break; case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.Compute_UnaryScalar(p); + MatrixComputeModel.KernelComputeUnaryScalar(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - MatrixComputeModel.Compute_Binary(p); + MatrixComputeModel.KernelComputeBinary(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.Compute_BinaryScalar(p); + MatrixComputeModel.KernelComputeBinaryScalar(p); break; default: break; @@ -400,7 +400,9 @@ namespace StgSharp.Mathematics.Numeric private static extern IntPtr GetCurrentThread(); [DllImport("kernel32.dll")] - private static extern nuint SetThreadAffinityMask(IntPtr hThread, nuint dwThreadAffinityMask); + private static extern nuint SetThreadAffinityMask( + IntPtr hThread, + nuint dwThreadAffinityMask); [LibraryImport("kernel32.dll")] [return: MarshalAs(UnmanagedType.Bool)] diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 72a19eed..3a34a6f1 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -140,7 +140,9 @@ typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_pane typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar); #define PNL_PROC(T_type, T_arch, T_op_name) T_type##_pnl_##T_op_name##_##T_arch - +#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_op_name) \ + INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \ + T_op_name)(MAT_PANEL_##T_arch##(T_type) *restrict ans) #define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_op_name) \ INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, \ T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \ @@ -223,8 +225,14 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, fill); #pragma region fma +DECLARE_PNL_PROC_ANS(float, sse, clear); +DECLARE_PNL_PROC_ANS(float, avx, clear); +DECLARE_PNL_PROC_ANS(float, 512, clear); + DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma); DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma); +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma); +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, 512, fma); #pragma endregion @@ -266,6 +274,7 @@ typedef struct sn_intrinsic { BUF_PROC_LRA f32_buffer_sub; KER_PROC_RA f32_buffer_transpose; PANEL_PROC f32_build_panel; + PANEL_PROC f32_clear_panel; VECTORNORMALIZEPROC f32_normalize_3; PNL_PROC_LRA f32_panel_fma; PANEL_PROC f32_store_panel; diff --git a/StgSharp.Native/sn-head/sn_matkernel.h b/StgSharp.Native/sn-head/sn_matkernel.h index 6c1b12fc..c322f983 100644 --- a/StgSharp.Native/sn-head/sn_matkernel.h +++ b/StgSharp.Native/sn-head/sn_matkernel.h @@ -117,7 +117,11 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL; int col_index, int row_index) /* Declarations for AVX float variant */ +DECLARE_BUILD_PANEL(sse, float); DECLARE_BUILD_PANEL(avx, float); +DECLARE_BUILD_PANEL(512, float); +DECLARE_STORE_PANEL(sse, float); DECLARE_STORE_PANEL(avx, float); +DECLARE_STORE_PANEL(512, float); #endif /* SN_MATKERNEL */ diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index b3ff326c..32b29092 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -1,5 +1,32 @@ #include "sn_intrinsic.h" +DECLARE_PNL_PROC_ANS(float, sse, clear) +{ + register __m128 zero = _mm_setzero_ps(); + PANEL_COL_VEC((*ans), 0, 0) = zero; + PANEL_COL_VEC((*ans), 1, 0) = zero; + PANEL_COL_VEC((*ans), 2, 0) = zero; + PANEL_COL_VEC((*ans), 3, 0) = zero; +} + +DECLARE_PNL_PROC_ANS(float, avx, clear) +{ + register __m256 zero = _mm256_setzero_ps(); + PANEL_COL_VEC((*ans), 0, 0) = zero; + PANEL_COL_VEC((*ans), 1, 0) = zero; + PANEL_COL_VEC((*ans), 2, 0) = zero; + PANEL_COL_VEC((*ans), 3, 0) = zero; +} + +DECLARE_PNL_PROC_ANS(float, 512, clear) +{ + register __m512 zero = _mm512_setzero_ps(); + PANEL_COL_VEC((*ans), 0, 0) = zero; + PANEL_COL_VEC((*ans), 1, 0) = zero; + PANEL_COL_VEC((*ans), 2, 0) = zero; + PANEL_COL_VEC((*ans), 3, 0) = zero; +} + // Matrix transpose implementations DECLARE_KER_PROC_RIGHT_ANS(float, sse, transpose) { diff --git a/StgSharp.Native/sn-src/k_panel.c b/StgSharp.Native/sn-src/k_panel.c index baa40211..95c302bc 100644 --- a/StgSharp.Native/sn-src/k_panel.c +++ b/StgSharp.Native/sn-src/k_panel.c @@ -41,6 +41,51 @@ DECLARE_BUILD_PANEL(avx, float) } } +DECLARE_BUILD_PANEL(sse, float) +{ + /* Single kernel -> 4 columns, each is one __m128 column vector. */ + const MAT_KERNEL(float) *src = matrix + (col_index * row_length + row_index); + + for (int pc = 0; pc < 4; ++pc) { + PANEL_COL_VEC((*panel), pc, 0) = src->xmm[pc]; + } +} + +DECLARE_BUILD_PANEL(512, float) +{ + const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; + const MAT_KERNEL(float) *k[4][4]; + + for (int kc = 0; kc < 4; ++kc) { + for (int kr = 0; kr < 4; ++kr) { + const int col = col_index + kc; + const int row = row_index + kr; + k[kc][kr] = (col < col_length && row < row_length) ? + (matrix + (col * row_length + row)) : + zero; + } + } + + /* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */ + for (int pc = 0; pc < 16; ++pc) { + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + + const __m128 r0 = k[blk_col][0]->xmm[kc_col]; + const __m128 r1 = k[blk_col][1]->xmm[kc_col]; + const __m128 r2 = k[blk_col][2]->xmm[kc_col]; + const __m128 r3 = k[blk_col][3]->xmm[kc_col]; + + const __m256 top = _mm256_set_m128(r1, r0); + const __m256 bot = _mm256_set_m128(r3, r2); + + __m512 col = _mm512_castps256_ps512(top); + col = _mm512_insertf32x8(col, bot, 1); + + PANEL_COL_VEC((*panel), pc, 0) = col; + } +} + DECLARE_STORE_PANEL(avx, float) { const int Q = 8; /* fixed for AVX float panel */ @@ -96,3 +141,49 @@ DECLARE_STORE_PANEL(avx, float) k_lb->xmm[kc_col] = bot128; } } + +DECLARE_STORE_PANEL(sse, float) +{ + MAT_KERNEL(float) *dst = matrix + (col_index * row_length + row_index); + + for (int pc = 0; pc < 4; ++pc) { + const __m128 col = PANEL_COL_VEC((*panel), pc, 0); + dst->xmm[pc] = col; + } +} + +DECLARE_STORE_PANEL(512, float) +{ + MAT_KERNEL(float) zero; + MAT_KERNEL(float) *k[4][4]; + + for (int kc = 0; kc < 4; ++kc) { + for (int kr = 0; kr < 4; ++kr) { + const int col = col_index + kc; + const int row = row_index + kr; + k[kc][kr] = (col < col_length && row < row_length) ? + (matrix + (col * row_length + row)) : + &zero; + } + } + + for (int pc = 0; pc < 16; ++pc) { + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + + const __m512 col = PANEL_COL_VEC((*panel), pc, 0); + + const __m256 top = _mm512_castps512_ps256(col); + const __m256 bot = _mm512_extractf32x8_ps(col, 1); + + const __m128 r0 = _mm256_castps256_ps128(top); + const __m128 r1 = _mm256_extractf128_ps(top, 1); + const __m128 r2 = _mm256_castps256_ps128(bot); + const __m128 r3 = _mm256_extractf128_ps(bot, 1); + + k[blk_col][0]->xmm[kc_col] = r0; + k[blk_col][1]->xmm[kc_col] = r1; + k[blk_col][2]->xmm[kc_col] = r2; + k[blk_col][3]->xmm[kc_col] = r3; + } +} diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index a13c8383..97ff5062 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -65,8 +65,10 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; + context->f32_clear_panel = PNL_PROC(float, sse, clear); + context->f32_build_panel = BUILD_PANEL(sse, float); context->f32_buffer_add = BUF_PROC(float, sse, add); - context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_panel_fma = PNL_PROC(float, sse, fma); context->f32_buffer_fill = BUF_PROC(float, sse, fill); context->variant = BUF_PROC(float, sse, scalar_mul); context->f32_buffer_sub = BUF_PROC(float, sse, sub); @@ -80,8 +82,10 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; + context->f32_clear_panel = PNL_PROC(float, avx, clear); + context->f32_build_panel = BUILD_PANEL(avx, float); context->f32_buffer_add = BUF_PROC(float, avx, add); - context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_panel_fma = PNL_PROC(float, avx, fma); context->f32_buffer_fill = BUF_PROC(float, avx, fill); context->variant = BUF_PROC(float, avx, scalar_mul); context->f32_buffer_sub = BUF_PROC(float, avx, sub); @@ -95,8 +99,10 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; + context->f32_clear_panel = PNL_PROC(float, sse, clear); + context->f32_build_panel = BUILD_PANEL(512, float); context->f32_buffer_add = BUF_PROC(float, 512, add); - context->f32_panel_fma = KER_PROC(float, sse, fma); + context->f32_panel_fma = PNL_PROC(float, 512, fma); context->f32_buffer_fill = BUF_PROC(float, 512, fill); context->variant = BUF_PROC(float, 512, scalar_mul); context->f32_buffer_sub = BUF_PROC(float, 512, sub); From 2c7277f3f9f7b1329ee3ff2e62f7a261da05c525 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 12 Dec 2025 18:00:49 +0800 Subject: [PATCH 3/4] working on matrix mul matrix, rebuilding task model defines in C# --- .../Numeric/MatrixCompute.Additional.cs | 27 +- .../Numeric/MatrixCompute.Basic.cs | 2 +- .../Numeric/MatrixComputeModel.cs | 228 +++-------------- .../Numeric/MatrixParallel.Publish.cs | 14 +- .../Numeric/MatrixParallelTaskPackage.cs | 23 +- .../Numeric/MatrixParallelThread.cs | 16 +- .../Numeric/MatrixParallelWrap.cs | 85 +------ StgSharp.Native/sn-src/dllmain.c | 2 +- StgSharp.Native/sn-src/k_panel.c | 12 +- StgSharp.Native/sn-src/load_glfw.c | 232 +++++++++--------- StgSharp.Native/sn-src/str_hash.c | 6 +- StgSharp.Native/sn-src/vector_dot.c | 20 +- 12 files changed, 223 insertions(+), 444 deletions(-) diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs index 9b3b2030..beccf2c2 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs @@ -29,6 +29,7 @@ using StgSharp.HighPerformance; using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; @@ -37,17 +38,23 @@ namespace StgSharp.Mathematics.Numeric public partial class MatrixCompute { - private static unsafe void ComputeFmaPanelF32( - MatrixKernel* left, - MatrixKernel* right, - MatrixKernel* ans, - ScalarPacket* scalar) + internal static unsafe void ComputeFmaPanelF32(MatrixParallelTaskPackage* p) { - const int eSize = 32; - int colum = scalar->Data(0); - int row = scalar->Data(1); - int k = - + MatrixPanel* leftPanel = MatrixPanel.Create(); + MatrixPanel* rightPanel = MatrixPanel.Create(); + MatrixPanel* ansPanel = MatrixPanel.Create(); + ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); + ref long lengthRef = ref Unsafe.As(ref p->PrimCount); + int i_length = p->PrimCount; + int j_length = p->SecCount; + int k_length = 0; + for (long i = 0; i < lengthRef; i++) + { + for (int k = 0; k < k_length; k++) { } + } + MatrixPanel.Destroy(leftPanel); + MatrixPanel.Destroy(rightPanel); + MatrixPanel.Destroy(ansPanel); } } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index 99bfe321..71bade0c 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -222,7 +222,7 @@ namespace StgSharp.Mathematics.Numeric package->PrimCount = left.KernelColumnLength; package->SecCount = left.KernelRowLength; package->Scalar = scalar; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); + package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationSpecial.SEQ_FMA); package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul; MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); diff --git a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs index a96db5e0..7878055f 100644 --- a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs @@ -34,7 +34,7 @@ namespace StgSharp.Mathematics.Numeric internal static class MatrixComputeModel { - internal static unsafe void BufferComputeBinary(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeBinary(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -47,7 +47,21 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelFactory.Release(p); } - internal static unsafe void BufferComputeNoOperatorScalar(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeBinaryScalar(MatrixParallelTaskPackage* p) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + ScalarPacket* scalar = p->Scalar; + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), + (void*)(p->Right + offset), + (void*)(p->Result + offset), + scalar, + count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void KernelComputeNoOperatorScalar(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -57,7 +71,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelFactory.Release(p); } - internal static unsafe void BufferComputeUnary(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeUnary(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -69,7 +83,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelFactory.Release(p); } - internal static unsafe void BufferComputeUnaryScalar(MatrixParallelTaskPackage* p) + internal static unsafe void KernelComputeUnaryScalar(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -81,201 +95,19 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelFactory.Release(p); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void KernelComputeBinary(MatrixParallelTaskPackage* p) - { - int eSize = p->ElementSize; - - // base pointer to matrix kernels - nint l = p->Left, r = p->Right, a = p->Result; - long - - // set beginning value to counters for primary offsets - lCount = (long)p->LeftPrimOffset * p->LeftPrimStride, - rCount = (long)p->RightPrimOffset * p->RightPrimStride, - aCount = (long)p->ResultPrimOffset * p->ResultPrimStride, - pCount = int.Min(p->PrimCount, p->PrimColumnCountInTile + p->PrimTileOffset) /*_count of tile may be too much*/, - //edge to reset counters - lLimit = pCount - p->LeftPrimOffset, - rLimit = pCount - p->RightPrimOffset, - aLimit = pCount - p->ResultPrimOffset; - delegate* unmanaged[Cdecl] op = - (delegate* unmanaged[Cdecl])p->ComputeHandle; - for (long i = p->PrimTileOffset % pCount; i < pCount; i++) - { - // update primary counters - // if reaches edge, reset to zero - lCount = i < lLimit ? lCount + p->LeftPrimStride : 0; - rCount = i < rLimit ? rCount + p->RightPrimStride : 0; - aCount = i < rLimit ? aCount + p->ResultPrimStride : 0; - - // get kernel column pointers - nint left = l + (nint)(lCount * eSize); - nint right = r + (nint)(rCount * eSize); - nint ans = a + (nint)(aCount * eSize); - long - lc = (long)p->LeftSecOffset * p->LeftSecStride, - rc = (long)p->RightSecOffset * p->RightSecStride, - ac = (long)p->ResultSecOffset * p->ResultSecStride, - sCount = int.Min(p->SecCount, p->PrimColumnCountInTile * p->SecTileOffset); - long - ll = sCount - p->LeftSecOffset, - rl = sCount - p->RightSecOffset, - al = sCount - p->ResultSecOffset; - for (long j = p->SecTileOffset % sCount; j < sCount; j++) - { - lc = j < ll ? lc + p->LeftSecStride : 0; - rc = j < rl ? rc + p->RightSecStride : 0; - ac = j < al ? ac + p->ResultSecStride : 0; - op(left + (nint)lc, right + (nint)rc, ans + (nint)ac); - } - } - MatrixParallelFactory.Release(p); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void KernelComputeBinaryScalar(MatrixParallelTaskPackage* p) - { - int eSize = p->ElementSize; - - // base pointer to matrix kernels - nint l = p->Left, r = p->Right, a = p->Result; - long - - // set beginning value to counters for primary offsets - lCount = (long)p->LeftPrimOffset * p->LeftPrimStride, - rCount = (long)p->RightPrimOffset * p->RightPrimStride, - aCount = (long)p->ResultPrimOffset * p->ResultPrimStride, - pCount = int.Min(p->PrimCount, p->PrimColumnCountInTile + p->PrimTileOffset) /*_count of tile may be too much*/, - //edge to reset counters - lLimit = pCount - p->LeftPrimOffset, - rLimit = pCount - p->RightPrimOffset, - aLimit = pCount - p->ResultPrimOffset; - delegate* unmanaged[Cdecl] op = - (delegate* unmanaged[Cdecl])p->ComputeHandle; - for (long i = p->PrimTileOffset % pCount; i < pCount; i++) - { - // update primary counters - // if reaches edge, reset to zero - lCount = i < lLimit ? lCount + p->LeftPrimStride : 0; - rCount = i < rLimit ? rCount + p->RightPrimStride : 0; - aCount = i < rLimit ? aCount + p->ResultPrimStride : 0; - - // get kernel column pointers - nint left = l + (nint)(lCount * eSize); - nint right = r + (nint)(rCount * eSize); - nint ans = a + (nint)(aCount * eSize); - long - lc = (long)p->LeftSecOffset * p->LeftSecStride, - rc = (long)p->RightSecOffset * p->RightSecStride, - ac = (long)p->ResultSecOffset * p->ResultSecStride, - sCount = int.Min(p->SecCount, p->PrimColumnCountInTile * p->SecTileOffset); - long - ll = sCount - p->LeftSecOffset, - rl = sCount - p->RightSecOffset, - al = sCount - p->ResultSecOffset; - for (long j = p->SecTileOffset % sCount; j < sCount; j++) - { - lc = j < ll ? lc + p->LeftSecStride : 0; - rc = j < rl ? rc + p->RightSecStride : 0; - ac = j < al ? ac + p->ResultSecStride : 0; - op(left + (nint)lc, right + (nint)rc, ans + (nint)ac, p->Scalar); - } - } - MatrixParallelFactory.Release(p); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void KernelComputeUnary(MatrixParallelTaskPackage* p) - { - int eSize = p->ElementSize; - - // base pointer to matrix kernels - nint r = p->Right, a = p->Result; - long - - // set beginning value to counters for primary offsets - rCount = (long)p->RightPrimOffset * p->RightPrimStride, - aCount = (long)p->ResultPrimOffset * p->ResultPrimStride, - pCount = int.Min(p->PrimCount, p->PrimColumnCountInTile + p->PrimTileOffset) /*_count of tile may be too much*/, - //edge to reset counters - rLimit = pCount - p->RightPrimOffset, - aLimit = pCount - p->ResultPrimOffset; - delegate* unmanaged[Cdecl] op = - (delegate* unmanaged[Cdecl])p->ComputeHandle; - for (long i = p->PrimTileOffset % pCount; i < pCount; i++) - { - // update primary counters - // if reaches edge, reset to zero - rCount = i < rLimit ? rCount + p->RightPrimStride : 0; - aCount = i < rLimit ? aCount + p->ResultPrimStride : 0; - - // get kernel column pointers - nint right = r + (nint)(rCount * eSize); - nint ans = a + (nint)(aCount * eSize); - long - rc = (long)p->RightSecOffset * p->RightSecStride, - ac = (long)p->ResultSecOffset * p->ResultSecStride, - sCount = int.Min(p->SecCount, p->PrimColumnCountInTile * p->SecTileOffset); - long - rl = sCount - p->RightSecOffset, - al = sCount - p->ResultSecOffset; - for (long j = p->SecTileOffset % sCount; j < sCount; j++) - { - rc = j < rl ? rc + p->RightSecStride : 0; - ac = j < al ? ac + p->ResultSecStride : 0; - op(right + (nint)rc, ans + (nint)ac); - } - } - MatrixParallelFactory.Release(p); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void KernelComputeUnaryScalar(MatrixParallelTaskPackage* p) - { - int eSize = p->ElementSize; - - // base pointer to matrix kernels - nint r = p->Right, a = p->Result; - long - - // set beginning value to counters for primary offsets - rCount = (long)p->RightPrimOffset * p->RightPrimStride, - aCount = (long)p->ResultPrimOffset * p->ResultPrimStride, - pCount = int.Min(p->PrimCount, p->PrimColumnCountInTile + p->PrimTileOffset) /*_count of tile may be too much*/, - //edge to reset counters - rLimit = pCount - p->RightPrimOffset, - aLimit = pCount - p->ResultPrimOffset; - delegate* unmanaged[Cdecl] op = - (delegate* unmanaged[Cdecl])p->ComputeHandle; - for (long i = p->PrimTileOffset % pCount; i < pCount; i++) - { - // update primary counters - // if reaches edge, reset to zero - rCount = i < rLimit ? rCount + p->RightPrimStride : 0; - aCount = i < rLimit ? aCount + p->ResultPrimStride : 0; - - // get kernel column pointers - nint right = r + (nint)(rCount * eSize); - nint ans = a + (nint)(aCount * eSize); - long - rc = (long)p->RightSecOffset * p->RightSecStride, - ac = (long)p->ResultSecOffset * p->ResultSecStride, - sCount = int.Min(p->SecCount, p->PrimColumnCountInTile * p->SecTileOffset); - long - rl = sCount - p->RightSecOffset, - al = sCount - p->ResultSecOffset; - for (long j = p->SecTileOffset % sCount; j < sCount; j++) - { - rc = j < rl ? rc + p->RightSecStride : 0; - ac = j < al ? ac + p->ResultSecStride : 0; - op(right + (nint)rc, ans + (nint)ac, p->Scalar); - } - } - MatrixParallelFactory.Release(p); - } - internal static unsafe void PanelComputeBinary(MatrixParallelTaskPackage* p) { } + internal static unsafe void SpecialCompute(MatrixParallelTaskPackage* p) + { + switch ((MatrixOperationSpecial)p->ComputeMode.OperationStyle) + { + case MatrixOperationSpecial.SEQ_FMA: + MatrixCompute.ComputeFmaPanelF32(p); + break; + default: + throw new NotSupportedException("The special matrix operation style is not supported."); + } + } + } } \ No newline at end of file diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs index 437efd04..de3f2b3d 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixParallel.Publish" // Project: StgSharp @@ -51,18 +51,15 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelTaskPackage* nonGeneric = package; switch (nonGeneric->ComputeMode.OperationStyle) { - case MatrixOperationStyle.PANEL_OP: - throw new NotImplementedException(); - case MatrixOperationStyle.BUFFER_OP: + case MatrixOperationStyle.BUFFER_INDEX: return ScheduleBufferTask(nonGeneric); - case MatrixOperationStyle.KERNEL_OP: - return ScheduleKernelTask(nonGeneric); default: throw new InvalidOperationException($"Unknown mode {nonGeneric->ComputeMode.OperationStyle}."); } } - private static unsafe MatrixParallelHandle ScheduleBufferTask(MatrixParallelTaskPackage* package) + private static unsafe MatrixParallelHandle ScheduleBufferTask( + MatrixParallelTaskPackage* package) { int primCount = package->PrimCount; int secCount = package->SecCount; @@ -113,7 +110,8 @@ namespace StgSharp.Mathematics.Numeric return handle; } - private static unsafe MatrixParallelHandle ScheduleKernelTask(MatrixParallelTaskPackage* package) + private static unsafe MatrixParallelHandle ScheduleKernelTask( + MatrixParallelTaskPackage* package) { int primCount = package->PrimCount; int secCount = package->SecCount; diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs index c6429ee1..71a9eb32 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixParallelTaskPackage" // Project: StgSharp @@ -52,12 +52,17 @@ namespace StgSharp.Mathematics.Numeric * matrix_op_mode; */ + public enum MatrixOperationSpecial : short + { + + SEQ_FMA = -1, + + } + public enum MatrixOperationStyle : short { - PANEL_OP = 1, - BUFFER_OP = 2, - KERNEL_OP = 3, + BUFFER_INDEX = 1, } @@ -79,6 +84,9 @@ namespace StgSharp.Mathematics.Numeric public MatrixOperationParam ParameterStyle = paramLayout; public MatrixOperationStyle OperationStyle = style; + public MatrixOpMode(MatrixOperationStyle style, MatrixOperationSpecial paramLayout) + : this(style, (MatrixOperationParam)paramLayout) { } + } [StructLayout(LayoutKind.Explicit, Size = 128)] @@ -86,9 +94,12 @@ namespace StgSharp.Mathematics.Numeric { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void Copy(MatrixParallelTaskPackage* source, MatrixParallelTaskPackage* target) + public static unsafe void Copy( + MatrixParallelTaskPackage* source, + MatrixParallelTaskPackage* target) { - Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackage), sizeof(MatrixParallelTaskPackage)); + Buffer.MemoryCopy(source, target, sizeof(MatrixParallelTaskPackage), + sizeof(MatrixParallelTaskPackage)); } #region source diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index 0afa39ef..d16e3422 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -85,26 +85,30 @@ namespace StgSharp.Mathematics.Numeric while (taskQueue.TryDequeue(out nint handle)) { MatrixParallelTaskPackage* p = (MatrixParallelTaskPackage*)handle; + if ((int)p->ComputeMode.ParameterStyle < 0) { + MatrixComputeModel.SpecialCompute(p); + } switch (p->ComputeMode.OperationStyle) { case MatrixOperationStyle.PANEL_OP: - break; - case MatrixOperationStyle.BUFFER_OP: switch (p->ComputeMode.ParameterStyle) { case MatrixOperationParam.RIGHT_ANS_PARAM: - MatrixComputeModel.BufferComputeUnary(p); + + // MatrixComputeModel.PanelComputeUnary(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - MatrixComputeModel.BufferComputeBinary(p); + MatrixComputeModel.PanelComputeBinary(p); break; case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.BufferComputeUnaryScalar(p); + + // MatrixComputeModel.PanelComputeUnaryScalar(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: break; case MatrixOperationParam.ANS_SCALAR_PARAM: - MatrixComputeModel.BufferComputeNoOperatorScalar(p); + + // MatrixComputeModel.PanelComputeNoOperatorScalar(p); break; default: break; diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs index 0bb04ec7..1c27d8ba 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixParallelWrap" // Project: StgSharp @@ -104,18 +104,12 @@ namespace StgSharp.Mathematics.Numeric while (tasks.TryDequeue(out nint handle)) { MatrixParallelTaskPackage* task = (MatrixParallelTaskPackage*)handle; - switch (task->ComputeMode.OperationStyle) + if ((int)task->ComputeMode.OperationStyle > 0) { - case MatrixOperationStyle.PANEL_OP: - break; - case MatrixOperationStyle.BUFFER_OP: - PublishBufferTaskToWorkers(task); - break; - case MatrixOperationStyle.KERNEL_OP: - PublishBufferTaskToWorkers(task); - break; - default: - break; + PublishBufferTaskToWorkers(task); + } else + { + throw new NotImplementedException(); } } } @@ -169,73 +163,6 @@ namespace StgSharp.Mathematics.Numeric // Console.WriteLine(msg); } - private void PublishKernelTaskToWorkers(MatrixParallelTaskPackage* package) - { - int primCount = package->PrimCount; - int secCount = package->SecCount; - ReadOnlySpan pool = Threads; - - if (pool.Length == 1) - { - pool[0].Tasks.Enqueue((nint)package); - return; - } - throw new NotImplementedException(); - /* - int threadCount = pool.Length; - int tilePerThreadLess = tileCount / threadCount; // tiles per thread without extra slice - int columnAfterBaseTile = tileCount % threadCount * tileWidth; // tile sliced into columns - int extraColumnPerThread = columnAfterBaseTile / threadCount; - int extraColumnCount = columnAfterBaseTile % threadCount; // columns to be sliced - - int total = primCount; - Stack taskBunch = new Stack(threadCount); - MatrixParallelTaskPackageNonGeneric* current = package; - foreach (MatrixParallelWrap wrap in pool.Wraps) - { - bool hasSlice = columnAfterBaseTile > 0; - bool isLargeSlice = extraColumnCount > 0; - - // _count tiles for wrap - int wrapWidth = wrap!.WrapTaskCapacity; - int columnFromSlicingCount = (hasSlice ? extraColumnPerThread : 0) * wrapWidth; - int baseColumn = wrapWidth * tilePerThreadLess * tileWidth; - int columnInWrap = baseColumn + columnFromSlicingCount + (isLargeSlice ? 1 : 0); - - // set tasks - if (columnInWrap == 0) - { -#if DEBUG - throw new InvalidOperationException("Zero tasks published."); -#else - break; -#endif - } - - // update current task - current->PrimColumnCountInTile = columnInWrap; - current->SecColumnCountInTile = current->SecCount; - current->SecTileOffset = 0; - - // remove tiles _count - extraColumnCount--; - columnAfterBaseTile -= columnFromSlicingCount; - total -= columnInWrap; - - // set to wrap and push - wrap.WrapTask = current; - - if (total <= 0) - { - break; - } - current = MatrixParallelFactory.FromExistPackage(current); - current->PrimTileOffset += columnInWrap; - } - /**/ - return; - } - #region wrap task internal unsafe Queue WrapTask { get; set; } = []; diff --git a/StgSharp.Native/sn-src/dllmain.c b/StgSharp.Native/sn-src/dllmain.c index 677a0b71..38b0014c 100644 --- a/StgSharp.Native/sn-src/dllmain.c +++ b/StgSharp.Native/sn-src/dllmain.c @@ -1,4 +1,4 @@ -#include "StgSharpNative.h" +#include "StgSharpNative.h" #include "sn_matkernel.h" #include diff --git a/StgSharp.Native/sn-src/k_panel.c b/StgSharp.Native/sn-src/k_panel.c index 95c302bc..4d3c8bb6 100644 --- a/StgSharp.Native/sn-src/k_panel.c +++ b/StgSharp.Native/sn-src/k_panel.c @@ -54,7 +54,7 @@ DECLARE_BUILD_PANEL(sse, float) DECLARE_BUILD_PANEL(512, float) { const MAT_KERNEL(float) *const zero = (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; - const MAT_KERNEL(float) *k[4][4]; + const MAT_KERNEL(float) * k[4][4]; for (int kc = 0; kc < 4; ++kc) { for (int kr = 0; kr < 4; ++kr) { @@ -68,8 +68,8 @@ DECLARE_BUILD_PANEL(512, float) /* Each panel column corresponds to a kernel column (0..3) within a block col (0..3). */ for (int pc = 0; pc < 16; ++pc) { - const int blk_col = pc >> 2; /* which kernel column block (0..3) */ - const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ const __m128 r0 = k[blk_col][0]->xmm[kc_col]; const __m128 r1 = k[blk_col][1]->xmm[kc_col]; @@ -155,7 +155,7 @@ DECLARE_STORE_PANEL(sse, float) DECLARE_STORE_PANEL(512, float) { MAT_KERNEL(float) zero; - MAT_KERNEL(float) *k[4][4]; + MAT_KERNEL(float) * k[4][4]; for (int kc = 0; kc < 4; ++kc) { for (int kr = 0; kr < 4; ++kr) { @@ -168,8 +168,8 @@ DECLARE_STORE_PANEL(512, float) } for (int pc = 0; pc < 16; ++pc) { - const int blk_col = pc >> 2; /* which kernel column block (0..3) */ - const int kc_col = pc & 3; /* column inside a 4x4 kernel */ + const int blk_col = pc >> 2; /* which kernel column block (0..3) */ + const int kc_col = pc & 3; /* column inside a 4x4 kernel */ const __m512 col = PANEL_COL_VEC((*panel), pc, 0); diff --git a/StgSharp.Native/sn-src/load_glfw.c b/StgSharp.Native/sn-src/load_glfw.c index f225f968..02f0ec0b 100644 --- a/StgSharp.Native/sn-src/load_glfw.c +++ b/StgSharp.Native/sn-src/load_glfw.c @@ -4,120 +4,120 @@ SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table) { - table->glfwCreateCursor = glfwCreateCursor; - table->glfwCreateStandardCursor = glfwCreateStandardCursor; - table->glfwCreateWindow = glfwCreateWindow; - table->glfwDefaultWindowHints = glfwDefaultWindowHints; - table->glfwDestroyCursor = glfwDestroyCursor; - table->glfwDestroyWindow = glfwDestroyWindow; - table->glfwExtensionSupported = glfwExtensionSupported; - table->glfwFocusWindow = glfwFocusWindow; - table->glfwGetClipboardString = glfwGetClipboardString; - table->glfwGetCurrentContext = glfwGetCurrentContext; - table->glfwGetCursorPos = glfwGetCursorPos; - table->glfwGetError = glfwGetError; - table->glfwGetFrameBufferSize = glfwGetFramebufferSize; - table->glfwGetGamepadName = glfwGetGamepadName; - table->glfwGetGamepadState = glfwGetGamepadState; - table->glfwGetGammaRamp = glfwGetGammaRamp; - table->glfwGetInputMode = glfwGetInputMode; - table->glfwGetJoystickAxes = glfwGetJoystickAxes; - table->glfwGetJoystickButtons = glfwGetJoystickButtons; - table->glfwGetJoystickGUID = glfwGetJoystickGUID; - table->glfwGetJoystickHats = glfwGetJoystickHats; - table->glfwGetJoystickName = glfwGetJoystickName; - table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer; - table->glfwGetKey = glfwGetKey; - table->glfwGetKeyName = glfwGetKeyName; - table->glfwGetKeyScancode = glfwGetKeyScancode; - table->glfwGetMonitorContentScale = glfwGetMonitorContentScale; - table->glfwGetMonitorName = glfwGetMonitorName; - table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize; - table->glfwGetMonitorPos = glfwGetMonitorPos; - table->glfwGetMonitors = glfwGetMonitors; - table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer; - table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea; - table->glfwGetMouseButton = glfwGetMouseButton; - table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor; - table->glfwGetProcAddress = glfwGetProcAddress; - table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions; - table->glfwGetTime = glfwGetTime; - table->glfwGetTimerFrequency = glfwGetTimerFrequency; - table->glfwGetTimerValue = glfwGetTimerValue; - table->glfwGetVersion = glfwGetVersion; - table->glfwGetVersionString = glfwGetVersionString; - table->glfwGetVideoMode = glfwGetVideoMode; - table->glfwGetVideoModes = glfwGetVideoModes; - table->glfwGetWindowAttrib = glfwGetWindowAttrib; - table->glfwGetWindowContentScale = glfwGetWindowContentScale; - table->glfwGetWindowFrameSize = glfwGetWindowFrameSize; - table->glfwGetWindowMonitor = glfwGetWindowMonitor; - table->glfwGetWindowOpacity = glfwGetWindowOpacity; - table->glfwGetWindowPos = glfwGetWindowPos; - table->glfwGetWindowSize = glfwGetWindowSize; - table->glfwGetWindowUserPointer = glfwGetWindowUserPointer; - table->glfwHideWindow = glfwHideWindow; - table->glfwIconifyWindow = glfwIconifyWindow; - table->glfwInit = glfwInit; - table->glfwInitHint = glfwInitHint; - table->glfwJoystickIsGamePad = glfwJoystickIsGamepad; - table->glfwJoystickPresent = glfwJoystickPresent; - table->glfwMakeContextCurrent = glfwMakeContextCurrent; - table->glfwMaximizeWindow = glfwMaximizeWindow; - table->glfwPollEvents = glfwPollEvents; - table->glfwPostEmptyEvent = glfwPostEmptyEvent; - table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported; - table->glfwRequestWindowAttention = glfwRequestWindowAttention; - table->glfwRestoreWindow = glfwRestoreWindow; - table->glfwSetCharCallback = glfwSetCharCallback; - table->glfwSetCharModsCallback = glfwSetCharModsCallback; - table->glfwSetClipboardString = glfwSetClipboardString; - table->glfwSetCursor = glfwSetCursor; - table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback; - table->glfwSetCursorPos = glfwSetCursorPos; - table->glfwSetCursorPosCallback = glfwSetCursorPosCallback; - table->glfwSetDropCallback = glfwSetDropCallback; - table->glfwSetErrorCallback = glfwSetErrorCallback; - table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback; - table->glfwSetGamma = glfwSetGamma; - table->glfwSetGammaRamp = glfwSetGammaRamp; - table->glfwSetInputMode = glfwSetInputMode; - table->glfwSetJoystickCallback = glfwSetJoystickCallback; - table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer; - table->glfwSetKeyCallback = glfwSetKeyCallback; - table->glfwSetMonitorCallback = glfwSetMonitorCallback; - table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer; - table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback; - table->glfwSetScrollCallback = glfwSetScrollCallback; - table->glfwSetTime = glfwSetTime; - table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio; - table->glfwSetWindowAttrib = glfwSetWindowAttrib; - table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback; - table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback; - table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback; - table->glfwSetWindowIcon = glfwSetWindowIcon; - table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback; - table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback; - table->glfwSetWindowMonitor = glfwSetWindowMonitor; - table->glfwSetWindowOpacity = glfwSetWindowOpacity; - table->glfwSetWindowPos = glfwSetWindowPos; - table->glfwSetWindowPosCallback = glfwSetWindowPosCallback; - table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback; - table->glfwSetWindowShouldClose = glfwSetWindowShouldClose; - table->glfwSetWindowSize = glfwSetWindowSize; - table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback; - table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits; - table->glfwSetWindowTitle = glfwSetWindowTitle; - table->glfwSetWindowUserPointer = glfwSetWindowUserPointer; - table->glfwShowWindow = glfwShowWindow; - table->glfwSwapBuffers = glfwSwapBuffers; - table->glfwSwapInterval = glfwSwapInterval; - table->glfwTerminate = glfwTerminate; - table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings; - table->glfwVulkanSupported = glfwVulkanSupported; - table->glfwWaitEvents = glfwWaitEvents; - table->glfwWaitEventsTimeout = glfwWaitEventsTimeout; - table->glfwWindowHint = glfwWindowHint; - table->glfwWindowHintString = glfwWindowHintString; - table->glfwWindowShouldClose = glfwWindowShouldClose; + table->glfwCreateCursor = glfwCreateCursor; + table->glfwCreateStandardCursor = glfwCreateStandardCursor; + table->glfwCreateWindow = glfwCreateWindow; + table->glfwDefaultWindowHints = glfwDefaultWindowHints; + table->glfwDestroyCursor = glfwDestroyCursor; + table->glfwDestroyWindow = glfwDestroyWindow; + table->glfwExtensionSupported = glfwExtensionSupported; + table->glfwFocusWindow = glfwFocusWindow; + table->glfwGetClipboardString = glfwGetClipboardString; + table->glfwGetCurrentContext = glfwGetCurrentContext; + table->glfwGetCursorPos = glfwGetCursorPos; + table->glfwGetError = glfwGetError; + table->glfwGetFrameBufferSize = glfwGetFramebufferSize; + table->glfwGetGamepadName = glfwGetGamepadName; + table->glfwGetGamepadState = glfwGetGamepadState; + table->glfwGetGammaRamp = glfwGetGammaRamp; + table->glfwGetInputMode = glfwGetInputMode; + table->glfwGetJoystickAxes = glfwGetJoystickAxes; + table->glfwGetJoystickButtons = glfwGetJoystickButtons; + table->glfwGetJoystickGUID = glfwGetJoystickGUID; + table->glfwGetJoystickHats = glfwGetJoystickHats; + table->glfwGetJoystickName = glfwGetJoystickName; + table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer; + table->glfwGetKey = glfwGetKey; + table->glfwGetKeyName = glfwGetKeyName; + table->glfwGetKeyScancode = glfwGetKeyScancode; + table->glfwGetMonitorContentScale = glfwGetMonitorContentScale; + table->glfwGetMonitorName = glfwGetMonitorName; + table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize; + table->glfwGetMonitorPos = glfwGetMonitorPos; + table->glfwGetMonitors = glfwGetMonitors; + table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer; + table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea; + table->glfwGetMouseButton = glfwGetMouseButton; + table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor; + table->glfwGetProcAddress = glfwGetProcAddress; + table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions; + table->glfwGetTime = glfwGetTime; + table->glfwGetTimerFrequency = glfwGetTimerFrequency; + table->glfwGetTimerValue = glfwGetTimerValue; + table->glfwGetVersion = glfwGetVersion; + table->glfwGetVersionString = glfwGetVersionString; + table->glfwGetVideoMode = glfwGetVideoMode; + table->glfwGetVideoModes = glfwGetVideoModes; + table->glfwGetWindowAttrib = glfwGetWindowAttrib; + table->glfwGetWindowContentScale = glfwGetWindowContentScale; + table->glfwGetWindowFrameSize = glfwGetWindowFrameSize; + table->glfwGetWindowMonitor = glfwGetWindowMonitor; + table->glfwGetWindowOpacity = glfwGetWindowOpacity; + table->glfwGetWindowPos = glfwGetWindowPos; + table->glfwGetWindowSize = glfwGetWindowSize; + table->glfwGetWindowUserPointer = glfwGetWindowUserPointer; + table->glfwHideWindow = glfwHideWindow; + table->glfwIconifyWindow = glfwIconifyWindow; + table->glfwInit = glfwInit; + table->glfwInitHint = glfwInitHint; + table->glfwJoystickIsGamePad = glfwJoystickIsGamepad; + table->glfwJoystickPresent = glfwJoystickPresent; + table->glfwMakeContextCurrent = glfwMakeContextCurrent; + table->glfwMaximizeWindow = glfwMaximizeWindow; + table->glfwPollEvents = glfwPollEvents; + table->glfwPostEmptyEvent = glfwPostEmptyEvent; + table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported; + table->glfwRequestWindowAttention = glfwRequestWindowAttention; + table->glfwRestoreWindow = glfwRestoreWindow; + table->glfwSetCharCallback = glfwSetCharCallback; + table->glfwSetCharModsCallback = glfwSetCharModsCallback; + table->glfwSetClipboardString = glfwSetClipboardString; + table->glfwSetCursor = glfwSetCursor; + table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback; + table->glfwSetCursorPos = glfwSetCursorPos; + table->glfwSetCursorPosCallback = glfwSetCursorPosCallback; + table->glfwSetDropCallback = glfwSetDropCallback; + table->glfwSetErrorCallback = glfwSetErrorCallback; + table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback; + table->glfwSetGamma = glfwSetGamma; + table->glfwSetGammaRamp = glfwSetGammaRamp; + table->glfwSetInputMode = glfwSetInputMode; + table->glfwSetJoystickCallback = glfwSetJoystickCallback; + table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer; + table->glfwSetKeyCallback = glfwSetKeyCallback; + table->glfwSetMonitorCallback = glfwSetMonitorCallback; + table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer; + table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback; + table->glfwSetScrollCallback = glfwSetScrollCallback; + table->glfwSetTime = glfwSetTime; + table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio; + table->glfwSetWindowAttrib = glfwSetWindowAttrib; + table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback; + table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback; + table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback; + table->glfwSetWindowIcon = glfwSetWindowIcon; + table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback; + table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback; + table->glfwSetWindowMonitor = glfwSetWindowMonitor; + table->glfwSetWindowOpacity = glfwSetWindowOpacity; + table->glfwSetWindowPos = glfwSetWindowPos; + table->glfwSetWindowPosCallback = glfwSetWindowPosCallback; + table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback; + table->glfwSetWindowShouldClose = glfwSetWindowShouldClose; + table->glfwSetWindowSize = glfwSetWindowSize; + table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback; + table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits; + table->glfwSetWindowTitle = glfwSetWindowTitle; + table->glfwSetWindowUserPointer = glfwSetWindowUserPointer; + table->glfwShowWindow = glfwShowWindow; + table->glfwSwapBuffers = glfwSwapBuffers; + table->glfwSwapInterval = glfwSwapInterval; + table->glfwTerminate = glfwTerminate; + table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings; + table->glfwVulkanSupported = glfwVulkanSupported; + table->glfwWaitEvents = glfwWaitEvents; + table->glfwWaitEventsTimeout = glfwWaitEventsTimeout; + table->glfwWindowHint = glfwWindowHint; + table->glfwWindowHintString = glfwWindowHintString; + table->glfwWindowShouldClose = glfwWindowShouldClose; } diff --git a/StgSharp.Native/sn-src/str_hash.c b/StgSharp.Native/sn-src/str_hash.c index 89794198..8662dab9 100644 --- a/StgSharp.Native/sn-src/str_hash.c +++ b/StgSharp.Native/sn-src/str_hash.c @@ -15,7 +15,7 @@ INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length) for (; ptr < end; ptr += 4) { hash = _mm_crc32_u32(hash, *(uint32_t *)ptr); } - for (; ptr < end; ptr++) { + for (; ptr < end; ptr++) { hash = _mm_crc32_u16(hash, *ptr); } } else { @@ -30,10 +30,10 @@ INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length) hash = _mm_crc32_u8(hash, hash64); hash = _mm_crc32_u8(hash, hash64 >> 32); for (; ptr < last_32; ptr += 4) { - hash = _mm_crc32_u32(hash,*(uint32_t *)ptr); + hash = _mm_crc32_u32(hash, *(uint32_t *)ptr); } for (; ptr < end; ptr += 2) { - hash = _mm_crc32_u8(hash,*ptr); + hash = _mm_crc32_u8(hash, *ptr); } } hash ^= hash >> 16; diff --git a/StgSharp.Native/sn-src/vector_dot.c b/StgSharp.Native/sn-src/vector_dot.c index 4d00414d..7b18cf27 100644 --- a/StgSharp.Native/sn-src/vector_dot.c +++ b/StgSharp.Native/sn-src/vector_dot.c @@ -15,7 +15,8 @@ INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose, SSE_INSERT_SIMPLE(1, 4)); } -INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) +INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, + __m128 *ans) { __m256 _double = _mm256_set_m128(vector[0], vector[0]); __m256 tmp = _mm256_dp_ps(transpose->ymm[0], _double, 0B11110001); @@ -25,7 +26,8 @@ INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 cons _mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices))); } -INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) +INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector, + __m128 *ans) { ans[0] = _mm_dp_ps(transpose->xmm[0], vector[0], 0B11110001); ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->xmm[1], vector[0], 0B11110001), @@ -44,24 +46,22 @@ INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 cons SSE_INSERT_SIMPLE(1, 4)); } -INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, __m128 *ans) +INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector, + __m128 *ans) { __m256 temp = transpose->ymm[0]; __m256 vector_stream = _mm256_set_m128(vector[1], vector[0]); __m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - result = _mm256_blend_ps( - result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010); + result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010); temp = transpose->ymm[1]; - result = _mm256_blend_ps( - result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100); + result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100); temp = _mm256_permute2f128_ps(temp, temp, 0B00000001); - result = _mm256_blend_ps( - result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000); + result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000); ans[0] = _mm256_castps256_ps128(result); ans[1] = _mm256_extractf128_ps(result, 1); ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1)); -} \ No newline at end of file +} From 88ed5a95c5189027e2d1913d1be3e55b150d410a Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:15:30 +0800 Subject: [PATCH 4/4] Mat mul mat is now on full function --- .../Internal/Intrinsic/IntrinsicContext.cs | 2 +- .../Numeric/MatrixCompute.Additional.cs | 33 +- .../Numeric/MatrixCompute.Basic.cs | 281 ------------------ StgSharp.Mathematics/Numeric/MatrixCompute.cs | 252 +++++++++++++++- .../Numeric/MatrixComputeModel.cs | 28 +- .../Numeric/MatrixParallel.Publish.cs | 77 +---- .../Numeric/MatrixParallel.cs | 2 +- .../Numeric/MatrixParallelTaskPackage.cs | 21 +- .../Numeric/MatrixParallelThread.cs | 37 +-- StgSharp.Native/sn-head/sn_matkernel.h | 4 +- StgSharp.Native/sn-src/k_basicop.c | 14 +- StgSharp.Native/sn-src/k_panel.c | 24 +- StgSharp.Native/sn-src/load_intrinsic.c | 5 +- StgSharpDebug/Program.cs | 65 ++-- 14 files changed, 357 insertions(+), 488 deletions(-) delete mode 100644 StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index 064daceb..40fb3f2e 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -44,7 +44,7 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] f32_buffer_scalar_mul; //f32_buffer_s_m public delegate* unmanaged[Cdecl] f32_buffer_sub; //f32_kernel_sub public delegate* unmanaged[Cdecl] f32_buffer_transpose; //f32_kernel_transpose - public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel + public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel public delegate* unmanaged[Cdecl] f32_clear_panel; public delegate* unmanaged[Cdecl] f32_normalize_3; //f32_normalize_3 public delegate* unmanaged[Cdecl] f32_panel_fma; //f32_panel_fma diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs index beccf2c2..24e7e2b7 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Additional.cs @@ -26,6 +26,7 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.HighPerformance; +using StgSharp.Internal; using System; using System.Collections.Generic; using System.Linq; @@ -40,17 +41,39 @@ namespace StgSharp.Mathematics.Numeric internal static unsafe void ComputeFmaPanelF32(MatrixParallelTaskPackage* p) { + int size = p->ElementSize; + MatrixKernel* leftBuffer = (MatrixKernel*)p->Left; + MatrixKernel* rightBuffer = (MatrixKernel*)p->Right; + MatrixKernel* ansBuffer = (MatrixKernel*)p->Answer; MatrixPanel* leftPanel = MatrixPanel.Create(); MatrixPanel* rightPanel = MatrixPanel.Create(); MatrixPanel* ansPanel = MatrixPanel.Create(); ref long offsetRef = ref Unsafe.As(ref p->PrimTileOffset); ref long lengthRef = ref Unsafe.As(ref p->PrimCount); - int i_length = p->PrimCount; - int j_length = p->SecCount; - int k_length = 0; - for (long i = 0; i < lengthRef; i++) + int ansWidth = p-> AnsPrimOffset; + int ansHeight = p->AnsSecOffset; + int commonK = p->LeftPrimOffset; + int blocksPerRow = ansWidth / size; + + for (long c = offsetRef; c < offsetRef + lengthRef; c++) { - for (int k = 0; k < k_length; k++) { } + int i = (int)(c / blocksPerRow) * size; + int j = (int)(c % blocksPerRow) * size; + NativeIntrinsic.Intrinsic.f32_clear_panel((MatrixPanel*)ansPanel); + for (int k = 0; k < commonK; k += size) + { + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixPanel*)leftPanel, leftBuffer, commonK, + ansHeight, k, i); + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixPanel*)rightPanel, rightBuffer, ansWidth, + commonK, j, k); + NativeIntrinsic.Intrinsic + .f32_panel_fma((MatrixPanel*)leftPanel, (MatrixPanel*)rightPanel, (MatrixPanel*)ansPanel); + } + NativeIntrinsic.Intrinsic + .f32_store_panel((MatrixPanel*)ansPanel, ansBuffer, ansWidth, + ansHeight, j, i); } MatrixPanel.Destroy(leftPanel); MatrixPanel.Destroy(rightPanel); diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs deleted file mode 100644 index 71bade0c..00000000 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ /dev/null @@ -1,281 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="MatrixCompute.Basic" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using StgSharp.Collections; -using StgSharp.HighPerformance; -using StgSharp.Internal; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations.Schema; -using System.Linq; -using System.Numerics; -using System.Security.AccessControl; -using System.Text; -using System.Threading.Tasks; - -namespace StgSharp.Mathematics.Numeric -{ - public static partial class MatrixCompute - { - - public static unsafe void Add(Matrix left, Matrix right, Matrix ans) - { - if (left.ColumnLength != right.ColumnLength || - left.RowLength != right.RowLength || - left.ColumnLength != ans.ColumnLength || - left.RowLength != ans.RowLength) { - throw new IndexOutOfRangeException("Matrix dimensions must agree."); - } - long count = (long)left.KernelColumnLength * left.KernelRowLength; - if (count <= 256) - { - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* rightPtr = right.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); - - return; - } else if (count <= 1024) - { - MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); - - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* rightPtr = right.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); - - MatrixParallel.ReturnParallelResources(ref pool); - return; - } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = (nint)left.Buffer; - package->Right = (nint)right.Buffer; - package->Result = (nint)ans.Buffer; - package->ElementSize = sizeof(float); - package->PrimCount = ans.KernelColumnLength; - package->SecCount = ans.KernelRowLength; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); - package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_add; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); - taskGroup.Dispose(); - return; - } - - public static unsafe void Fill(Matrix ans, float value) - { - long count = (long)ans.KernelColumnLength * ans.KernelRowLength; - ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket(); - scalar->Data(0) = value; - if (count <= 256) - { - NativeIntrinsic.Intrinsic.f32_buffer_fill(ans.Buffer, scalar, count); - return; - } else if (count <= 1024) - { - MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); - NativeIntrinsic.Intrinsic.f32_buffer_fill(ans.Buffer, scalar, count); - MatrixParallel.ReturnParallelResources(ref pool); - return; - } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Result = (nint)ans.Buffer; - package->ElementSize = sizeof(float); - package->PrimCount = ans.KernelColumnLength; - package->SecCount = ans.KernelRowLength; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.ANS_SCALAR_PARAM); - package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_fill; - package->Scalar = scalar; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); - MatrixParallelFactory.Release(scalar); - taskGroup.Dispose(); - } - - public static unsafe void Mul(Matrix left, Matrix right, Matrix ans) - { - // For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN - // ColumnLength = number of columns (N), RowLength = number of rows (M) - if (left.ColumnLength != right.RowLength || - ans.ColumnLength != right.ColumnLength || - ans.RowLength != left.RowLength) - { - throw new IndexOutOfRangeException("Matrix dimensions must agree."); - } - long count = (long)left.KernelColumnLength * left.KernelRowLength; - if (count <= 1024) - { - MatrixParallelThread[] pool = null; - if (count > 256) { - pool = MatrixParallel.LeadParallel(1); - } - MatrixPanel* leftPanel = MatrixPanel.Create(); - MatrixPanel* rightPanel = MatrixPanel.Create(); - MatrixPanel* ansPanel = MatrixPanel.Create(); - int size = MatrixPanel.KernelSideCount(); - for (int i = 0; i < left.RowLength; i += size) - { - for (int j = 0; j < right.ColumnLength; j += size) - { - NativeIntrinsic.Intrinsic.f32_clear_panel((MatrixPanel*)ansPanel); - for (int k = 0; k < left.ColumnLength; k += size) - { - NativeIntrinsic.Intrinsic - .f32_build_panel((MatrixKernel*)left.Buffer, - (MatrixPanel*)leftPanel, - left.ColumnLength, left.RowLength, i, - k); - NativeIntrinsic.Intrinsic - .f32_build_panel((MatrixKernel*)right.Buffer, - (MatrixPanel*)rightPanel, - right.ColumnLength, right.RowLength, k, - j); - NativeIntrinsic.Intrinsic - .f32_panel_fma((MatrixPanel*)leftPanel, - (MatrixPanel*)rightPanel, - (MatrixPanel*)ansPanel); - } - NativeIntrinsic.Intrinsic - .f32_store_panel((MatrixKernel*)ans.Buffer, - (MatrixPanel*)ansPanel, ans.ColumnLength, - ans.RowLength, i, j); - } - } - MatrixPanel.Destroy(leftPanel); - MatrixPanel.Destroy(rightPanel); - MatrixPanel.Destroy(ansPanel); - if (count > 256) { - MatrixParallel.ReturnParallelResources(ref pool!); - } - return; - } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = (nint)left.Buffer; - package->Result = (nint)ans.Buffer; - package->ElementSize = sizeof(float); - package->PrimCount = left.KernelColumnLength; - package->SecCount = left.KernelRowLength; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); - package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); - taskGroup.Dispose(); - return; - } - - public static unsafe void Mul(Matrix left, float right, Matrix ans) - { - if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { - throw new IndexOutOfRangeException("Matrix dimensions must agree."); - } - long count = (long)left.KernelColumnLength * left.KernelRowLength; - ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket(); - scalar->Data(0) = right; - if (count <= 256) - { - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); - - return; - } else if (count <= 1024) - { - MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); - - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); - - MatrixParallel.ReturnParallelResources(ref pool); - return; - } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Right = (nint)left.Buffer; - package->Result = (nint)ans.Buffer; - package->ElementSize = sizeof(float); - package->PrimCount = left.KernelColumnLength; - package->SecCount = left.KernelRowLength; - package->Scalar = scalar; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationSpecial.SEQ_FMA); - package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); - taskGroup.Dispose(); - MatrixParallelFactory.Release(scalar); - return; - } - - public static unsafe void Sub(Matrix left, Matrix right, Matrix ans) - { - if (left.ColumnLength != right.ColumnLength || - left.RowLength != right.RowLength || - left.ColumnLength != ans.ColumnLength || - left.RowLength != ans.RowLength) { - throw new IndexOutOfRangeException("Matrix dimensions must agree."); - } - long count = (long)left.KernelColumnLength * left.KernelRowLength; - if (count <= 256) - { - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* rightPtr = right.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); - - return; - } else if (count <= 1024) - { - MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); - - MatrixKernel* leftPtr = left.Buffer; - MatrixKernel* rightPtr = right.Buffer; - MatrixKernel* ansPtr = ans.Buffer; - NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); - - MatrixParallel.ReturnParallelResources(ref pool); - return; - } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); - package->Left = (nint)left.Buffer; - package->Right = (nint)right.Buffer; - package->Result = (nint)ans.Buffer; - package->ElementSize = sizeof(float); - package->PrimCount = left.KernelColumnLength; - package->SecCount = left.KernelRowLength; - package->ComputeMode = new MatrixOpMode(MatrixOperationStyle.BUFFER_OP, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); - package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_sub; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); - taskGroup.Dispose(); - return; - } - - } -} diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.cs index 829d6fc1..587df5c4 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.cs @@ -25,13 +25,263 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Collections; +using StgSharp.HighPerformance; +using StgSharp.Internal; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; using System.Linq; +using System.Numerics; +using System.Security.AccessControl; using System.Text; using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public static partial class MatrixCompute { } + public static partial class MatrixCompute + { + + public static unsafe void Add(Matrix left, Matrix right, Matrix ans) + { + if (left.ColumnLength != right.ColumnLength || + left.RowLength != right.RowLength || + left.ColumnLength != ans.ColumnLength || + left.RowLength != ans.RowLength) { + throw new IndexOutOfRangeException("Matrix dimensions must agree."); + } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + if (count <= 256) + { + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* rightPtr = right.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); + + return; + } else if (count <= 1024) + { + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* rightPtr = right.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_add(leftPtr, rightPtr, ansPtr, count); + + MatrixParallel.ReturnParallelResources(ref pool); + return; + } + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Left = (nint)left.Buffer; + package->Right = (nint)right.Buffer; + package->Answer = (nint)ans.Buffer; + package->ElementSize = sizeof(float); + package->PrimCount = ans.KernelColumnLength; + package->SecCount = ans.KernelRowLength; + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); + package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_add; + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.Dispose(); + return; + } + + public static unsafe void Fill(Matrix ans, float value) + { + long count = (long)ans.KernelColumnLength * ans.KernelRowLength; + ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket(); + scalar->Data(0) = value; + if (count <= 256) + { + NativeIntrinsic.Intrinsic.f32_buffer_fill(ans.Buffer, scalar, count); + return; + } else if (count <= 1024) + { + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + NativeIntrinsic.Intrinsic.f32_buffer_fill(ans.Buffer, scalar, count); + MatrixParallel.ReturnParallelResources(ref pool); + return; + } + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Answer = (nint)ans.Buffer; + package->ElementSize = sizeof(float); + package->PrimCount = ans.KernelColumnLength; + package->SecCount = ans.KernelRowLength; + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.ANS_SCALAR_PARAM); + package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_fill; + package->Scalar = scalar; + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + MatrixParallelFactory.Release(scalar); + taskGroup.Dispose(); + } + + public static unsafe void Mul(Matrix left, Matrix right, Matrix ans) + { + // For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN + // ColumnLength = number of columns (N), RowLength = number of rows (M) + if (left.ColumnLength != right.RowLength || + ans.ColumnLength != right.ColumnLength || + ans.RowLength != left.RowLength) + { + throw new IndexOutOfRangeException("Matrix dimensions must agree."); + } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + int size = MatrixPanel.KernelSideCount(); + if (count <= 1024) + { + MatrixParallelThread[] pool = null; + if (count > 256) { + pool = MatrixParallel.LeadParallel(1); + } + MatrixPanel* leftPanel = MatrixPanel.Create(); + MatrixPanel* rightPanel = MatrixPanel.Create(); + MatrixPanel* ansPanel = MatrixPanel.Create(); + for (int i = 0; i < ans.KernelRowLength; i += size) + { + for (int j = 0; j < ans.KernelColumnLength; j += size) + { + NativeIntrinsic.Intrinsic.f32_clear_panel((MatrixPanel*)ansPanel); + + // Console.WriteLine($"{i}{j}"); + for (int k = 0; k < left.KernelColumnLength; k += size) + { + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixPanel*)leftPanel, + (MatrixKernel*)left.Buffer, + left.KernelColumnLength, + left.KernelRowLength, i, k); + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixPanel*)rightPanel, + (MatrixKernel*)right.Buffer, + right.KernelColumnLength, + right.KernelRowLength, k, j); + NativeIntrinsic.Intrinsic + .f32_panel_fma((MatrixPanel*)leftPanel, + (MatrixPanel*)rightPanel, + (MatrixPanel*)ansPanel); + } + NativeIntrinsic.Intrinsic + .f32_store_panel((MatrixPanel*)ansPanel, + (MatrixKernel*)ans.Buffer, + ans.KernelColumnLength, ans.KernelRowLength, + i, j); + } + } + MatrixPanel.Destroy(leftPanel); + MatrixPanel.Destroy(rightPanel); + MatrixPanel.Destroy(ansPanel); + if (count > 256) { + MatrixParallel.ReturnParallelResources(ref pool!); + } + return; + } + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Left = (nint)left.Buffer; + package->Right = (nint)right.Buffer; + package->Answer = (nint)ans.Buffer; + package->AnsPrimOffset = ans.KernelRowLength; + package->AnsSecOffset = ans.KernelColumnLength; + package->LeftPrimOffset = left.KernelColumnLength; + package->PrimCount = ans.KernelColumnLength / size; + package->SecCount = ans.KernelRowLength / size; + package->ElementSize = size; + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA); + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.Dispose(); + return; + } + + public static unsafe void Mul(Matrix left, float right, Matrix ans) + { + if (left.ColumnLength != ans.ColumnLength || left.RowLength != ans.RowLength) { + throw new IndexOutOfRangeException("Matrix dimensions must agree."); + } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + ScalarPacket* scalar = MatrixParallelFactory.CreateScalarPacket(); + scalar->Data(0) = right; + if (count <= 256) + { + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); + + return; + } else if (count <= 1024) + { + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul(leftPtr, ansPtr, scalar, count); + + MatrixParallel.ReturnParallelResources(ref pool); + return; + } + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Right = (nint)left.Buffer; + package->Answer = (nint)ans.Buffer; + package->ElementSize = sizeof(float); + package->PrimCount = left.KernelColumnLength; + package->SecCount = left.KernelRowLength; + package->Scalar = scalar; + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); + package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_scalar_mul; + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.Dispose(); + MatrixParallelFactory.Release(scalar); + return; + } + + public static unsafe void Sub(Matrix left, Matrix right, Matrix ans) + { + if (left.ColumnLength != right.ColumnLength || + left.RowLength != right.RowLength || + left.ColumnLength != ans.ColumnLength || + left.RowLength != ans.RowLength) { + throw new IndexOutOfRangeException("Matrix dimensions must agree."); + } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + if (count <= 256) + { + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* rightPtr = right.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); + + return; + } else if (count <= 1024) + { + MatrixParallelThread[] pool = MatrixParallel.LeadParallel(1); + + MatrixKernel* leftPtr = left.Buffer; + MatrixKernel* rightPtr = right.Buffer; + MatrixKernel* ansPtr = ans.Buffer; + NativeIntrinsic.Intrinsic.f32_buffer_sub(leftPtr, rightPtr, ansPtr, count); + + MatrixParallel.ReturnParallelResources(ref pool); + return; + } + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + package->Left = (nint)left.Buffer; + package->Right = (nint)right.Buffer; + package->Answer = (nint)ans.Buffer; + package->ElementSize = sizeof(float); + package->PrimCount = left.KernelColumnLength; + package->SecCount = left.KernelRowLength; + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); + package->ComputeHandle = (IntPtr)NativeIntrinsic.Intrinsic.f32_buffer_sub; + MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + + MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.Dispose(); + return; + } + + } } diff --git a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs index 7878055f..ef479c26 100644 --- a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs @@ -34,7 +34,7 @@ namespace StgSharp.Mathematics.Numeric internal static class MatrixComputeModel { - internal static unsafe void KernelComputeBinary(MatrixParallelTaskPackage* p) + internal static unsafe void BufferComputeBinary(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -42,12 +42,12 @@ namespace StgSharp.Mathematics.Numeric ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), (void*)(p->Right + offset), - (void*)(p->Result + offset), + (void*)(p->Answer + offset), count); MatrixParallelFactory.Release(p); } - internal static unsafe void KernelComputeBinaryScalar(MatrixParallelTaskPackage* p) + internal static unsafe void BufferComputeBinaryScalar(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); @@ -55,57 +55,55 @@ namespace StgSharp.Mathematics.Numeric ScalarPacket* scalar = p->Scalar; ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), (void*)(p->Right + offset), - (void*)(p->Result + offset), + (void*)(p->Answer + offset), scalar, count); MatrixParallelFactory.Release(p); } - internal static unsafe void KernelComputeNoOperatorScalar(MatrixParallelTaskPackage* p) + internal static unsafe void BufferComputeNoOperatorScalar(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Result + offset), + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Answer + offset), p->Scalar, count); MatrixParallelFactory.Release(p); } - internal static unsafe void KernelComputeUnary(MatrixParallelTaskPackage* p) + internal static unsafe void BufferComputeUnary(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); long offset = Unsafe.As(ref p->PrimTileOffset) * size; ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Result + offset), + (void*)(p->Answer + offset), count); MatrixParallelFactory.Release(p); } - internal static unsafe void KernelComputeUnaryScalar(MatrixParallelTaskPackage* p) + internal static unsafe void BufferComputeUnaryScalar(MatrixParallelTaskPackage* p) { int size = p->ElementSize * 16; long count = Unsafe.As(ref p->PrimCount); long offset = Unsafe.As(ref p->PrimTileOffset) * size; ((delegate* unmanaged[Cdecl]< void*, void*, ScalarPacket*, long, void>)p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Result + offset), + (void*)(p->Answer + offset), p->Scalar, count); MatrixParallelFactory.Release(p); } - internal static unsafe void PanelComputeBinary(MatrixParallelTaskPackage* p) { } - - internal static unsafe void SpecialCompute(MatrixParallelTaskPackage* p) + internal static unsafe void SpecialBufferCompute(MatrixParallelTaskPackage* p) { - switch ((MatrixOperationSpecial)p->ComputeMode.OperationStyle) + switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle) { case MatrixOperationSpecial.SEQ_FMA: MatrixCompute.ComputeFmaPanelF32(p); break; default: - throw new NotSupportedException("The special matrix operation style is not supported."); + break; } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs index de3f2b3d..a082d737 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs @@ -51,7 +51,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallelTaskPackage* nonGeneric = package; switch (nonGeneric->ComputeMode.OperationStyle) { - case MatrixOperationStyle.BUFFER_INDEX: + case MatrixIndexStyle.BUFFER_INDEX: return ScheduleBufferTask(nonGeneric); default: throw new InvalidOperationException($"Unknown mode {nonGeneric->ComputeMode.OperationStyle}."); @@ -110,80 +110,5 @@ namespace StgSharp.Mathematics.Numeric return handle; } - private static unsafe MatrixParallelHandle ScheduleKernelTask( - MatrixParallelTaskPackage* package) - { - int primCount = package->PrimCount; - int secCount = package->SecCount; - int tileWidth = (1024 / secCount) + 1; - int tileCount = (primCount + tileWidth - 1) / tileWidth; - - MatrixParallelThread[] pool = LeadParallel(tileCount); - /* - if (pool.Length == 0) - { - // no need - // LeadParallel has contained capacity check - } - /**/ - MatrixParallelHandle handle = new MatrixParallelHandle(pool); - if (handle.Wraps.Length == 1) - { - handle.Wraps[0].WrapTask.Enqueue((nint)package); - return handle; - } - int threadCount = pool.Length; - int tilePerThreadLess = tileCount / threadCount; // tiles per thread without extra slice - int columnAfterBaseTile = tileCount % threadCount * tileWidth; // tile sliced into columns - int extraColumnPerThread = columnAfterBaseTile / threadCount; - int extraColumnCount = columnAfterBaseTile % threadCount; // columns to be sliced - - int total = primCount; - MatrixParallelTaskPackage* current = package; - foreach (MatrixParallelWrap wrap in handle.Wraps) - { - bool hasSlice = columnAfterBaseTile > 0; - bool isLargeSlice = extraColumnCount > 0; - - // _count tiles for wrap - int wrapWidth = wrap!.WrapTaskCapacity/**/; - int columnFromSlicingCount = (hasSlice ? extraColumnPerThread : 0) * wrapWidth; - int baseColumn = wrapWidth * tilePerThreadLess * tileWidth; - int columnInWrap = baseColumn + columnFromSlicingCount + (isLargeSlice ? 1 : 0); - - // set tasks - if (columnInWrap == 0) - { -#if DEBUG - throw new InvalidOperationException("Zero tasks published."); -#else - break; -#endif - } - - // update current task - current->PrimColumnCountInTile = columnInWrap; - current->SecColumnCountInTile = current->SecCount; - current->SecTileOffset = 0; - - // remove tiles _count - extraColumnCount--; - columnAfterBaseTile -= columnFromSlicingCount; - total -= columnInWrap; - - // set to wrap and push - wrap.WrapTask.Enqueue((nint)current); - - if (total <= 0) - { - break; - } - current = MatrixParallelFactory.FromExistPackage(current); - current->PrimTileOffset += columnInWrap; - } - - return handle; - } - } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.cs index b2619463..10a3efbe 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.cs @@ -34,7 +34,7 @@ using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public static class MatrixParallel + public static partial class MatrixParallel { private static bool _meaningful; diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs index 71a9eb32..4db2273f 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelTaskPackage.cs @@ -59,7 +59,7 @@ namespace StgSharp.Mathematics.Numeric } - public enum MatrixOperationStyle : short + public enum MatrixIndexStyle : short { BUFFER_INDEX = 1, @@ -78,13 +78,14 @@ namespace StgSharp.Mathematics.Numeric } [StructLayout(LayoutKind.Sequential)] - public struct MatrixOpMode(MatrixOperationStyle style, MatrixOperationParam paramLayout) + public struct MatrixOpMode(MatrixIndexStyle style, MatrixOperationParam paramLayout) { - public MatrixOperationParam ParameterStyle = paramLayout; - public MatrixOperationStyle OperationStyle = style; + public MatrixIndexStyle OperationStyle = style; - public MatrixOpMode(MatrixOperationStyle style, MatrixOperationSpecial paramLayout) + public MatrixOperationParam ParameterStyle = paramLayout; + + public MatrixOpMode(MatrixIndexStyle style, MatrixOperationSpecial paramLayout) : this(style, (MatrixOperationParam)paramLayout) { } } @@ -113,7 +114,7 @@ namespace StgSharp.Mathematics.Numeric #region ans // Group 2: Result kernel base address + reserved - [FieldOffset(16)] public IntPtr Result; // 8 + [FieldOffset(16)] public IntPtr Answer; // 8 [FieldOffset(24)] public int ElementSize; [FieldOffset(28)] public int ReservedPtr0; // 4 (extension/reserved) @@ -139,10 +140,10 @@ namespace StgSharp.Mathematics.Numeric #region ans enum - [FieldOffset(64)]public int ResultPrimOffset; // 4 - [FieldOffset(68)]public int ResultPrimStride; // 4 - [FieldOffset(72)]public int ResultSecOffset; // 4 - [FieldOffset(76)] public int ResultSecStride; // 4 + [FieldOffset(64)]public int AnsPrimOffset; // 4 + [FieldOffset(68)]public int AnsPrimStride; // 4 + [FieldOffset(72)]public int AnsSecOffset; // 4 + [FieldOffset(76)] public int AnsSecStride; // 4 #endregion diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index d16e3422..64e01834 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -85,49 +85,28 @@ namespace StgSharp.Mathematics.Numeric while (taskQueue.TryDequeue(out nint handle)) { MatrixParallelTaskPackage* p = (MatrixParallelTaskPackage*)handle; - if ((int)p->ComputeMode.ParameterStyle < 0) { - MatrixComputeModel.SpecialCompute(p); + if ((short)p->ComputeMode.ParameterStyle < 0) { + MatrixComputeModel.SpecialBufferCompute(p); } switch (p->ComputeMode.OperationStyle) { - case MatrixOperationStyle.PANEL_OP: + case MatrixIndexStyle.BUFFER_INDEX: switch (p->ComputeMode.ParameterStyle) { case MatrixOperationParam.RIGHT_ANS_PARAM: - - // MatrixComputeModel.PanelComputeUnary(p); + MatrixComputeModel.BufferComputeUnary(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - MatrixComputeModel.PanelComputeBinary(p); + MatrixComputeModel.BufferComputeBinary(p); break; case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - - // MatrixComputeModel.PanelComputeUnaryScalar(p); + MatrixComputeModel.BufferComputeUnaryScalar(p); break; case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: + MatrixComputeModel.BufferComputeBinaryScalar(p); break; case MatrixOperationParam.ANS_SCALAR_PARAM: - - // MatrixComputeModel.PanelComputeNoOperatorScalar(p); - break; - default: - break; - } - break; - case MatrixOperationStyle.KERNEL_OP: - switch (p->ComputeMode.ParameterStyle) - { - case MatrixOperationParam.RIGHT_ANS_PARAM: - MatrixComputeModel.KernelComputeUnary(p); - break; - case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.KernelComputeUnaryScalar(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - MatrixComputeModel.KernelComputeBinary(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.KernelComputeBinaryScalar(p); + MatrixComputeModel.BufferComputeNoOperatorScalar(p); break; default: break; diff --git a/StgSharp.Native/sn-head/sn_matkernel.h b/StgSharp.Native/sn-head/sn_matkernel.h index c322f983..52b58d11 100644 --- a/StgSharp.Native/sn-head/sn_matkernel.h +++ b/StgSharp.Native/sn-head/sn_matkernel.h @@ -110,9 +110,9 @@ extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL; MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \ int col_index, int row_index) #define DECLARE_STORE_PANEL(arch, element) \ - INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_KERNEL(element) *restrict matrix, \ - MAT_PANEL_##arch(element) \ + INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_PANEL_##arch(element) \ const *const panel, \ + MAT_KERNEL(element) *restrict matrix, \ int col_length, int row_length, \ int col_index, int row_index) diff --git a/StgSharp.Native/sn-src/k_basicop.c b/StgSharp.Native/sn-src/k_basicop.c index 32b29092..07d2ef06 100644 --- a/StgSharp.Native/sn-src/k_basicop.c +++ b/StgSharp.Native/sn-src/k_basicop.c @@ -12,19 +12,17 @@ DECLARE_PNL_PROC_ANS(float, sse, clear) DECLARE_PNL_PROC_ANS(float, avx, clear) { register __m256 zero = _mm256_setzero_ps(); - PANEL_COL_VEC((*ans), 0, 0) = zero; - PANEL_COL_VEC((*ans), 1, 0) = zero; - PANEL_COL_VEC((*ans), 2, 0) = zero; - PANEL_COL_VEC((*ans), 3, 0) = zero; + for (size_t i = 0; i < 8; i++) { + PANEL_COL_VEC((*ans), i, 0) = zero; + } } DECLARE_PNL_PROC_ANS(float, 512, clear) { register __m512 zero = _mm512_setzero_ps(); - PANEL_COL_VEC((*ans), 0, 0) = zero; - PANEL_COL_VEC((*ans), 1, 0) = zero; - PANEL_COL_VEC((*ans), 2, 0) = zero; - PANEL_COL_VEC((*ans), 3, 0) = zero; + for (size_t i = 0; i < 16; i++) { + PANEL_COL_VEC((*ans), i, 0) = zero; + } } // Matrix transpose implementations diff --git a/StgSharp.Native/sn-src/k_panel.c b/StgSharp.Native/sn-src/k_panel.c index 4d3c8bb6..916a6f23 100644 --- a/StgSharp.Native/sn-src/k_panel.c +++ b/StgSharp.Native/sn-src/k_panel.c @@ -8,16 +8,16 @@ DECLARE_BUILD_PANEL(avx, float) const int has_right = (col_index + 1) < col_length; const int has_bottom = (row_index + 1) < row_length; - const MAT_KERNEL(float) *const k_lt = matrix + (col_index * row_length + row_index); + const MAT_KERNEL(float) *const k_lt = matrix + (row_index * col_length + col_index); const MAT_KERNEL(float) *const k_rt = - has_right ? (matrix + ((col_index + 1) * row_length + row_index)) : + has_right ? (matrix + (row_index * col_length + (col_index + 1))) : (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; const MAT_KERNEL(float) *const k_lb = - has_bottom ? (matrix + (col_index * row_length + (row_index + 1))) : + has_bottom ? (matrix + ((row_index + 1) * col_length + col_index)) : (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; const MAT_KERNEL(float) *const k_rb = (has_right && has_bottom) ? - (matrix + ((col_index + 1) * row_length + (row_index + 1))) : + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : (const MAT_KERNEL(float) *)&SN_ZERO_KERNEL; register __m128 top128, bot128; @@ -44,7 +44,7 @@ DECLARE_BUILD_PANEL(avx, float) DECLARE_BUILD_PANEL(sse, float) { /* Single kernel -> 4 columns, each is one __m128 column vector. */ - const MAT_KERNEL(float) *src = matrix + (col_index * row_length + row_index); + const MAT_KERNEL(float) *src = matrix + (row_index * col_length + col_index); for (int pc = 0; pc < 4; ++pc) { PANEL_COL_VEC((*panel), pc, 0) = src->xmm[pc]; @@ -61,7 +61,7 @@ DECLARE_BUILD_PANEL(512, float) const int col = col_index + kc; const int row = row_index + kr; k[kc][kr] = (col < col_length && row < row_length) ? - (matrix + (col * row_length + row)) : + (matrix + (row * col_length + col)) : zero; } } @@ -97,19 +97,19 @@ DECLARE_STORE_PANEL(avx, float) const int has_bottom = (row_index + 1) < row_length; MAT_KERNEL(float) *const k_lt = (col_index < col_length && row_index < row_length) ? - (matrix + (col_index * row_length + row_index)) : + (matrix + (row_index * col_length + col_index)) : &zero; MAT_KERNEL(float) *const k_rt = has_right && (row_index < row_length) ? - (matrix + ((col_index + 1) * row_length + row_index)) : + (matrix + (row_index * col_length + (col_index + 1))) : &zero; MAT_KERNEL(float) *const k_lb = has_bottom && (col_index < col_length) ? - (matrix + (col_index * row_length + (row_index + 1))) : + (matrix + ((row_index + 1) * col_length + col_index)) : &zero; MAT_KERNEL(float) *const k_rb = (has_right && has_bottom) ? - (matrix + ((col_index + 1) * row_length + (row_index + 1))) : + (matrix + ((row_index + 1) * col_length + (col_index + 1))) : &zero; register __m128 top128, bot128; @@ -144,7 +144,7 @@ DECLARE_STORE_PANEL(avx, float) DECLARE_STORE_PANEL(sse, float) { - MAT_KERNEL(float) *dst = matrix + (col_index * row_length + row_index); + MAT_KERNEL(float) *dst = matrix + (row_index * col_length + col_index); for (int pc = 0; pc < 4; ++pc) { const __m128 col = PANEL_COL_VEC((*panel), pc, 0); @@ -162,7 +162,7 @@ DECLARE_STORE_PANEL(512, float) const int col = col_index + kc; const int row = row_index + kr; k[kc][kr] = (col < col_length && row < row_length) ? - (matrix + (col * row_length + row)) : + (matrix + (row * col_length + col)) : &zero; } } diff --git a/StgSharp.Native/sn-src/load_intrinsic.c b/StgSharp.Native/sn-src/load_intrinsic.c index 97ff5062..22b93f11 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.c +++ b/StgSharp.Native/sn-src/load_intrinsic.c @@ -65,6 +65,7 @@ void load_intrinsic_sse(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; + context->f32_store_panel = STORE_PANEL(sse, float); context->f32_clear_panel = PNL_PROC(float, sse, clear); context->f32_build_panel = BUILD_PANEL(sse, float); context->f32_buffer_add = BUF_PROC(float, sse, add); @@ -82,6 +83,7 @@ void load_intrinsic_avx(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; + context->f32_store_panel = STORE_PANEL(avx, float); context->f32_clear_panel = PNL_PROC(float, avx, clear); context->f32_build_panel = BUILD_PANEL(avx, float); context->f32_buffer_add = BUF_PROC(float, avx, add); @@ -99,7 +101,8 @@ void load_intrinsic_512(sn_intrinsic *context) context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; context->index_pair = index_pair_sse; - context->f32_clear_panel = PNL_PROC(float, sse, clear); + context->f32_store_panel = STORE_PANEL(512, float); + context->f32_clear_panel = PNL_PROC(float, 512, clear); context->f32_build_panel = BUILD_PANEL(512, float); context->f32_buffer_add = BUF_PROC(float, 512, add); context->f32_panel_fma = PNL_PROC(float, 512, fma); diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 66b23bbb..11299973 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="Program" // Project: StgSharp @@ -53,71 +53,44 @@ namespace StgSharp // HlsfStressTest.Run(); - /* + /**/ HybridLayerSegregatedFitAllocator hlsf; unchecked { hlsf = new((nuint)(64 * (ulong)10000 * 10000)); } - Matrix m = Matrix.FromHlsf(1000, 1000, hlsf); + Matrix m = Matrix.FromHlsf(400, 400, hlsf); MatrixCompute.Fill(m, 1.0f); + Matrix n = Matrix.FromHlsf(400, 400, hlsf); + MatrixCompute.Fill(n, 1.0f); + Matrix p = Matrix.FromHlsf(400, 400, hlsf); - Console.WriteLine(); - Console.WriteLine(); - Console.WriteLine(); - Matrix n = Matrix.FromHlsf(1000, 1000, hlsf); - MatrixCompute.Fill(n, 0.0f); - for (int i = 0; i < 5; i++) + // MatrixCompute.Fill(p, 0.0f); + /* + for (int i = 0; i < 10; i++) { - for (int j = 0; j < 2; j++) + MatrixCompute.Add(n, m, n); + for (int j = 0; j < 5; j++) { - MatrixCompute.Add(n, m, n); - - // Console.WriteLine($"{i} {j}"); + for (int k = 0; k < 5; k++) { + Console.Write($"{n[j, k]:0.0E+00} \t"); + } + Console.WriteLine(); } + Console.WriteLine($"{i}"); GC.Collect(); } - - MatrixCompute.Mul(n, 12, n); + /**/ + MatrixCompute.Mul(n, m, p); for (int i = 0; i < 20; i++) { for (int j = 0; j < 20; j++) { - Console.Write($"{n[i, j]:0.0E+00} \t"); + Console.Write($"{p[i, j]:0.0E+00} \t"); } Console.WriteLine(); } /**/ // --- Logarithmic time sequence test: print seconds at each emitted span --- - - // Configure a simple sequence: - // initial slice = 0.5s, total = 5s, growth multiplier = 2, min slice = 0.75s - TimeSequence seq = - /**/ - TimeSequence.CreateLogarithmic(initialSliceSeconds:0.5, totalSeconds:5.0, - growthMultiplier:2.0, minSliceSeconds:0.75); - /* - TimeSequence.CreateLinear(5, 1); - /**/ - - // Bind to default time source provider. - TimeSpanProvider provider = new TimeSpanProvider(seq, TimeSpanProvider.DefaultProvider); - - // Print the second mark when a span is emitted. - provider.TimeSpanRefreshed += (s, e) => { - // CurrentSecond is PreviousFrameTick / Frequency - Console.WriteLine($"span @ {provider.CurrentSecond:0.000}s"); - }; - - // Start the time source and wait until sequence ends. - TimeSpanProvider.DefaultProvider.StartProvidingTime(); - - while (!provider.IsEnded) { - provider.WaitNextSpan(); - } - - TimeSpanProvider.DefaultProvider.StopProvidingTime(); - - // end of log-time sequence test } catch (Exception ex) {