From 631bc2b6e89cfc732773fc075abbe0084837e980 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:52:17 +0800 Subject: [PATCH] working on matrix mul matrix --- .../HighPerformance/MatrixKernel.cs | 11 +- .../HighPerformance/MatrixPanel.cs | 270 +++++------------- .../Internal/Intrinsic/IntrinsicContext.cs | 4 +- .../Intrinsic/intrinsicContextProvider.cs | 2 +- .../Numeric/MatrixCompute.Basic.cs | 33 ++- .../Numeric/TaskPackageUsage.md | 0 StgSharp.Native/sn-src/k_fma.c | 43 +++ cache.txt | 9 - 8 files changed, 151 insertions(+), 221 deletions(-) delete mode 100644 StgSharp.Mathematics/Numeric/TaskPackageUsage.md diff --git a/StgSharp.Common/HighPerformance/MatrixKernel.cs b/StgSharp.Common/HighPerformance/MatrixKernel.cs index a460dd1a..4535487f 100644 --- a/StgSharp.Common/HighPerformance/MatrixKernel.cs +++ b/StgSharp.Common/HighPerformance/MatrixKernel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixKernel" // Project: StgSharp @@ -32,9 +32,16 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { + public unsafe struct MatrixKernel + { + + private byte Head; + + } + [StructLayout(LayoutKind.Sequential)] [InlineArray(16)] - public unsafe struct MatrixKernel where T: unmanaged, INumber + public unsafe struct MatrixKernel where T : unmanaged, INumber { private T c0r0; diff --git a/StgSharp.Common/HighPerformance/MatrixPanel.cs b/StgSharp.Common/HighPerformance/MatrixPanel.cs index c400c8c9..2e09dff3 100644 --- a/StgSharp.Common/HighPerformance/MatrixPanel.cs +++ b/StgSharp.Common/HighPerformance/MatrixPanel.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixPanel" // Project: StgSharp @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Internal; using System; using System.Numerics; using System.Runtime.CompilerServices; @@ -32,267 +33,130 @@ using System.Runtime.InteropServices; namespace StgSharp.HighPerformance { - #pragma warning disable CA1000 - - public interface IMatrixPanel : IDisposable where T : unmanaged,INumber + public unsafe struct MatrixPanel { - ref T this[int column, int row] { get; } - - static abstract int MaxElementCount { get; } - - static abstract int ElementSideLength { get; } - - static abstract nuint Size { get; } - - void Dispose(); + private byte Head; } - /// - /// Buffer for SSE targeted matrix panel, size of a panel is 4x4 elements, and no smaller than - /// 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixPanelSSE : IMatrixPanel where T : unmanaged, INumber + // Single POD-style panel: ZMM-compatible size. Alignment handled by allocator, not by the struct. + public unsafe struct MatrixPanel where T : unmanaged, INumber { - private T element; + public const int AlignmentBytes = 64; - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixPanelSSE() { } + internal T Head; // first element of the panel buffer public ref T this[int column, int row] { - get - { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => ref Unsafe.Add(ref Head, row * ElementSideLength + column); } public static int ElementSideLength { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(4, 16 / sizeof(T)); + get => Math.Max(8, AlignmentBytes / sizeof(T)); + } public static int MaxElementCount { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixPanelSSE* Create() - { - MatrixPanelSSE* kernel = (MatrixPanelSSE*)NativeMemory.AlignedAlloc(Size, 16); - return kernel; + get => ElementSideLength * ElementSideLength; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixPanelSSE* kernel) + public static nuint Size { - NativeMemory.AlignedFree(kernel); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (nuint)(MaxElementCount * sizeof(T)); } - public void Dispose() - { - fixed (MatrixPanelSSE* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M128 GetVectorChunk(int column, int chunk) - { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); - } - - // SIMD access helpers (column-major) - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 16 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M128 value) - { - GetVectorChunk(column, chunk) = value; - } - - } - - /// - /// Buffer for AVX targeted matrix panel, size of a panel is 4x4 elements, and no smaller than - /// 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixPanelAVX : IMatrixPanel where T : unmanaged, INumber - { - - private T element; - - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixPanelAVX() { } - - public ref T this[int column, int row] + public Span Elements { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); + ref T first = ref Head; + unsafe + { + void* p = Unsafe.AsPointer(ref first); + return new Span(p, MaxElementCount); + } } } - public static int ElementSideLength + // Allocate aligned memory for one panel; caller must free via Destroy. + public static unsafe MatrixPanel* Create() { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(8, 32 / sizeof(T)); - - public static unsafe int MaxElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixPanelAVX* Create() - { - MatrixPanelAVX* kernel = (MatrixPanelAVX*)NativeMemory.AlignedAlloc(Size, 32); - return kernel; + return (MatrixPanel*)NativeMemory.AlignedAlloc(Size, AlignmentBytes); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixPanelAVX* kernel) + public static unsafe void Destroy(MatrixPanel* panel) { - NativeMemory.AlignedFree(kernel); + NativeMemory.AlignedFree(panel); } - public void Dispose() + // View an existing pointer; caller guarantees lifetime/alignment. + public static unsafe MatrixPanel* FromPointer(void* ptr) { - fixed (MatrixPanelAVX* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } + return (MatrixPanel*)Align64(ptr); } + /// + /// Number of 4x4 kernels that fit in a panel for the given ISA and element T. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M256 GetVectorChunk(int column, int chunk) + internal static int KernelCapacity() { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + int perSide = KernelSideCount(); + return perSide * perSide; } - // SIMD access helpers for AVX (column-major) + /// + /// How many 4x4 kernels fit along a single panel dimension for the given ISA. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 32 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M256 value) + internal static int KernelSideCount() { - GetVectorChunk(column, chunk) = value; - } - - } - - /// - /// Buffer for AVX512 targeted matrix panel, size of a panel is 4x4 elements, and no smaller - /// than 16 bytes. - /// - /// - /// Type of matrix element. - /// - public unsafe struct MatrixComputeKernelAVX512 : IMatrixPanel - where T : unmanaged, INumber - { - - private T element; - - [Obsolete("Compute kernel should not be created from new(), use Create instead", true)] - public MatrixComputeKernelAVX512() { } - - public ref T this[int column, int row] - { - get + return NativeIntrinsic.IntrinsicLevel switch { - int index = row * ElementSideLength + column; - return ref Unsafe.Add(ref element, index); - } - } - - public static int ElementSideLength - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = int.Max(8, 64 / sizeof(T)); - - public static unsafe int MaxElementCount - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get; - } = ElementSideLength * ElementSideLength; - - public static unsafe nuint Size { get; } = (nuint)(MaxElementCount * sizeof(T)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ChunkCount() => (ElementSideLength + LanesPerVector() - 1) / LanesPerVector(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixComputeKernelAVX512* Create() - { - MatrixComputeKernelAVX512* kernel = (MatrixComputeKernelAVX512*)NativeMemory.AlignedAlloc(Size, 64); - return kernel; + IntrinsicLevel.SSE => KernelSideCountSse(), + IntrinsicLevel.AVX2 => KernelSideCountAvx2(), + IntrinsicLevel.AVX512 => KernelSideCountAvx512(), + _ => 0 + }; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Destroy(MatrixComputeKernelAVX512* kernel) - { - NativeMemory.AlignedFree(kernel); - } + private static unsafe nuint Align64(void* p) => ((nuint)p + (AlignmentBytes - 1)) & ~(nuint)(AlignmentBytes - 1); - public void Dispose() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int KernelSideCountAvx2() { - fixed (MatrixComputeKernelAVX512* thisPtr = &this) { - NativeMemory.AlignedFree(thisPtr); - } + const int minQ = 8; + int lanes = 32 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref M512 GetVectorChunk(int column, int chunk) + private static int KernelSideCountAvx512() { - int lanes = LanesPerVector(); - int rowStart = chunk * lanes; - ref T colBase = ref this[column, 0]; - return ref Unsafe.As(ref Unsafe.Add(ref colBase, rowStart)); + const int minQ = 8; + int lanes = 64 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } - // SIMD access helpers for AVX-512 (column-major) [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int LanesPerVector() => 64 / sizeof(T); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetVectorChunk(int column, int chunk, in M512 value) + private static int KernelSideCountSse() { - GetVectorChunk(column, chunk) = value; + const int minQ = 4; + int lanes = 16 / sizeof(T); + int side = Math.Max(lanes, minQ); + return side / 4; } -#pragma warning restore CA1000 + } -} \ No newline at end of file +} diff --git a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs index e3830279..53c1ffdb 100644 --- a/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs +++ b/StgSharp.Common/Internal/Intrinsic/IntrinsicContext.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="IntrinsicContext" // Project: StgSharp @@ -44,7 +44,7 @@ namespace StgSharp.Internal.Intrinsic public delegate* unmanaged[Cdecl] f32_buffer_scalar_mul; //f32_buffer_s_m public delegate* unmanaged[Cdecl] f32_buffer_sub; //f32_kernel_sub public delegate* unmanaged[Cdecl] f32_buffer_transpose; //f32_kernel_transpose - public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel + public delegate* unmanaged[Cdecl] f32_build_panel; //f32_build_panel public delegate* unmanaged[Cdecl] f32_normalize_3; //f32_normalize_3 public delegate* unmanaged[Cdecl] f32_panel_fma; //f32_panel_fma public delegate* unmanaged[Cdecl] f32_store_panel; //f32_store_panel diff --git a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs index 2fabc0a3..eaf92a40 100644 --- a/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs +++ b/StgSharp.Common/Internal/Intrinsic/intrinsicContextProvider.cs @@ -59,7 +59,7 @@ namespace StgSharp.Internal { case 0: - // Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform")); + Dialogue.PostError(new PlatformNotSupportedException("No SIMD instructions supported on current hardware platform")); Environment.Exit(-1); break; default: diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs index 52cec027..a0c05ae5 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Basic.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixCompute.Basic" // Project: StgSharp @@ -30,6 +30,7 @@ using StgSharp.HighPerformance; using StgSharp.Internal; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations.Schema; using System.Linq; using System.Numerics; using System.Security.AccessControl; @@ -124,11 +125,35 @@ namespace StgSharp.Mathematics.Numeric Matrix right, Matrix ans) { - if (left.RowLength != right.ColumnLength || - left.RowLength != ans.RowLength || - right.ColumnLength != ans.ColumnLength) { + // For matrix multiplication (left: MxK) * (right: KxN) => ans: MxN + // ColumnLength = number of columns (N), RowLength = number of rows (M) + if (left.ColumnLength != right.RowLength || + ans.ColumnLength != right.ColumnLength || + ans.RowLength != left.RowLength) + { throw new IndexOutOfRangeException("Matrix dimensions must agree."); } + long count = (long)left.KernelColumnLength * left.KernelRowLength; + if (count <= 256) + { + MatrixPanel* leftPanel = MatrixPanel.Create(); + MatrixPanel* rightPanel = MatrixPanel.Create(); + MatrixPanel* ansPanel = MatrixPanel.Create(); + int size = MatrixPanel.KernelSideCount(); + for (int i = 0; i < left.RowLength; i += size) + { + for (int j = 0; j < right.ColumnLength; j += size) + { + for (int k = 0; k < left.ColumnLength; k += size) { + NativeIntrinsic.Intrinsic + .f32_build_panel((MatrixKernel*)left.Buffer, + (MatrixPanel*)leftPanel, + left.ColumnLength, left.RowLength, i, + k); + } + } + } + } throw new NotImplementedException(); } diff --git a/StgSharp.Mathematics/Numeric/TaskPackageUsage.md b/StgSharp.Mathematics/Numeric/TaskPackageUsage.md deleted file mode 100644 index e69de29b..00000000 diff --git a/StgSharp.Native/sn-src/k_fma.c b/StgSharp.Native/sn-src/k_fma.c index d5e8510c..11ad5d05 100644 --- a/StgSharp.Native/sn-src/k_fma.c +++ b/StgSharp.Native/sn-src/k_fma.c @@ -52,6 +52,49 @@ DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, sse, fma) PANEL_COL_VEC((*ans), 3, 0) = c3; } +// ans = left * right + ans +// right is not transposed +DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx_fma, fma) +{ + static const __m256i idx0 = { 1, 1, 1, 1, 1, 1, 1, 1 }; + + for (int col = 0; col < 8; ++col) { + register __m256 bcol = _mm256_load_ps((float *)&right->vec[col][0]); + register __m256 ccol = _mm256_load_ps((float *)&ans->vec[col][0]); + register __m256i idx = _mm256_setzero_si256(); + register __m256i increament = _mm256_load_si256(&idx0); + + register __m256 a0 = _mm256_load_ps((float *)&left->vec[0][0]); + register __m256 a1 = _mm256_load_ps((float *)&left->vec[1][0]); + register __m256 a2 = _mm256_load_ps((float *)&left->vec[2][0]); + register __m256 a3 = _mm256_load_ps((float *)&left->vec[3][0]); + + ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + + a0 = _mm256_load_ps((float *)&left->vec[4][0]); + a1 = _mm256_load_ps((float *)&left->vec[5][0]); + a2 = _mm256_load_ps((float *)&left->vec[6][0]); + a3 = _mm256_load_ps((float *)&left->vec[7][0]); + + ccol = _mm256_fmadd_ps(a0, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a1, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a2, _mm256_permutevar8x32_ps(bcol, idx), ccol); + idx = _mm256_add_epi32(idx, increament); + ccol = _mm256_fmadd_ps(a3, _mm256_permutevar8x32_ps(bcol, idx), ccol); + + _mm256_store_ps((float *)&ans->vec[col][0], ccol); + } +} + // ans = left * right + ans // right is not transposed DECLARE_PNL_PROC_LEFT_RIGHT_ANS(float, avx, fma) diff --git a/cache.txt b/cache.txt index c5e7fe66..e69de29b 100644 --- a/cache.txt +++ b/cache.txt @@ -1,9 +0,0 @@ -public const uint GL_ACTIVE_SUBROUTINES = 0x8DE5 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORMS = 0x8DE6 -public const uint GL_MAX_SUBROUTINES = 0x8DE7 -public const uint GL_MAX_SUBROUTINE_UNIFORM_LOCATIONS = 0x8DE8 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORM_LOCATIONS = 0x8E47 -public const uint GL_ACTIVE_SUBROUTINE_MAX_LENGTH = 0x8E48 -public const uint GL_ACTIVE_SUBROUTINE_UNIFORM_MAX_LENGTH = 0x8E49 -public const uint GL_NUM_COMPATIBLE_SUBROUTINES = 0x8E4A -public const uint GL_COMPATIBLE_SUBROUTINES = 0x8E4B \ No newline at end of file