diff --git a/StgSharp.Common/Mathematics/Linear/MatrixParallel.Execute.cs b/StgSharp.Common/Mathematics/Linear/MatrixParallel.Execute.cs index 1d9b4a90..5d30dc10 100644 --- a/StgSharp.Common/Mathematics/Linear/MatrixParallel.Execute.cs +++ b/StgSharp.Common/Mathematics/Linear/MatrixParallel.Execute.cs @@ -31,6 +31,7 @@ using StgSharp.HighPerformance; using System; using System.Collections.Generic; +using System.Formats.Asn1; using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -42,95 +43,150 @@ namespace StgSharp.Mathematics { [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute(MatrixKernel* right, MatrixKernel* ans, IntPtr computeHandle) + internal static unsafe void Compute_Binary(MatrixParallelTaskPackage* p) { - ((delegate*)computeHandle)(right, ans); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute( - MatrixKernel* left, - MatrixKernel* right, - MatrixKernel* ans, - IntPtr computeHandle) - { - ((delegate*)computeHandle)(left, right, ans); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute( - MatrixKernel* right, - MatrixKernel* ans, - ScalarDataPacket* scalar, - IntPtr computeHandle) - { - ((delegate*)computeHandle)(right, ans, scalar); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void Compute( - MatrixKernel* left, - MatrixKernel* right, - MatrixKernel* ans, - ScalarDataPacket* scalar, - IntPtr computeHandle) - { - ((delegate*)computeHandle)(left, right, ans, scalar); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void ComputeRow( - MatrixKernel* right, - MatrixKernel* ans, - IntPtr computeHandle, - MatrixParallelTaskPackage* task) - { - long l = task->LeftEnum.secStride, r = task->RightEnum.secStride, a = task->ResultEnum.secStride, li = 0, ri = 0, ai = 0; - for (int i = 0; i < task->secCount; i++) { - ((delegate*)computeHandle)(right + ri, ans + ai); + MatrixKernel* l = p->Left, r = p->Right, a = p->Result; + long + lCount = p->LeftPrimOffset * p->LeftPrimStride, + rCount = p->RightPrimOffset * p->RightPrimStride, + aCount = p->ResultPrimOffset * p->ResultPrimStride, + pCount = p->PrimCount; + long + lLimit = pCount - p->LeftPrimOffset, + rLimit = pCount - p->RightPrimOffset, + aLimit = pCount - p->ResultPrimOffset; + delegate* unmanaged[Cdecl] op = + (delegate* unmanaged[Cdecl])p->ComputeHandle; + for (long i = 0; i < pCount; i++) + { + lCount = i < lLimit ? lCount + p->LeftPrimOffset : 0; + rCount = i < rLimit ? rCount + p->RightPrimOffset : 0; + aCount = i < rLimit ? aCount + p->ResultPrimOffset : 0; + MatrixKernel* left = l + lCount; + MatrixKernel* right = r + rCount; + MatrixKernel* ans = a + aCount; + long + lc = p->LeftSecOffset * p->LeftSecStride, + rc = p->RightSecOffset * p->RightSecStride, + ac = p->ResultSecOffset * p->ResultSecStride, + sCount = p->SecCount; + long + ll = sCount - p->LeftSecOffset, + rl = sCount - p->RightSecOffset, + al = sCount - p->ResultSecOffset; + for (int j = 0; j < sCount; j++) + { + lc = j < ll ? lc + p->LeftSecOffset : 0; + rc = j < rl ? rc + p->RightSecOffset : 0; + ac = j < al ? ac + p->ResultSecOffset : 0; + op(left + lc, right + rc, ans + ac); + } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void ComputeRow( - MatrixKernel* right, - MatrixKernel* ans, - ScalarDataPacket* scalar, - IntPtr computeHandle, - MatrixParallelTaskPackage* task) + internal static unsafe void Compute_BinaryScalar(MatrixParallelTaskPackage* p) { - long l = task->LeftEnum.secStride, r = task->RightEnum.secStride, a = task->ResultEnum.secStride, li = 0, ri = 0, ai = 0; - for (int i = 0; i < task->secCount; i++) { - ((delegate*)computeHandle)(right + ri, ans + ai, scalar); + MatrixKernel* l = p->Left, r = p->Right, a = p->Result; + long + lCount = p->LeftPrimOffset * p->LeftPrimStride, + rCount = p->RightPrimOffset * p->RightPrimStride, + aCount = p->ResultPrimOffset * p->ResultPrimStride, + pCount = p->PrimCount; + long + lLimit = pCount - p->LeftPrimOffset, + rLimit = pCount - p->RightPrimOffset, + aLimit = pCount - p->ResultPrimOffset; + delegate* unmanaged[Cdecl] op = + (delegate* unmanaged[Cdecl])p->ComputeHandle; + ScalarPacket* scalar = p->Scalar; + for (long i = 0; i < pCount; i++) + { + lCount = i < lLimit ? lCount + p->LeftPrimOffset : 0; + rCount = i < rLimit ? rCount + p->RightPrimOffset : 0; + aCount = i < rLimit ? aCount + p->ResultPrimOffset : 0; + MatrixKernel* left = l + lCount; + MatrixKernel* right = r + rCount; + MatrixKernel* ans = a + aCount; + long + lc = p->LeftSecOffset * p->LeftSecStride, + rc = p->RightSecOffset * p->RightSecStride, + ac = p->ResultSecOffset * p->ResultSecStride, + sCount = p->SecCount; + long + ll = sCount - p->LeftSecOffset, + rl = sCount - p->RightSecOffset, + al = sCount - p->ResultSecOffset; + for (int j = 0; j < sCount; j++) + { + lc = j < ll ? lc + p->LeftSecOffset : 0; + rc = j < rl ? rc + p->RightSecOffset : 0; + ac = j < al ? ac + p->ResultSecOffset : 0; + op(left + lc, right + rc, ans + ac, scalar); + } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void ComputeRow( - MatrixKernel* left, - MatrixKernel* right, - MatrixKernel* ans, - IntPtr computeHandle, - MatrixParallelTaskPackage* task) + internal static unsafe void Compute_Unary(MatrixParallelTaskPackage* p) { - long l = task->LeftEnum.secStride, r = task->RightEnum.secStride, a = task->ResultEnum.secStride, li = 0, ri = 0, ai = 0; - for (int i = 0; i < task->secCount; i++) { - ((delegate*)computeHandle)(left + li, right + ri, ans + ai); + MatrixKernel* s = p->Right, a = p->Result; + long rCount = p->RightPrimOffset * p->RightPrimStride, + aCount = p->ResultPrimOffset * p->ResultPrimStride, + pCount = p->PrimCount; + long rLimit = pCount - p->RightPrimOffset, + aLimit = pCount - p->ResultPrimOffset; + delegate* unmanaged[Cdecl] op = + (delegate* unmanaged[Cdecl])p->ComputeHandle; + for (long i = 0; i < pCount; i++) + { + rCount = i < rLimit ? rCount + p->RightPrimOffset : 0; + aCount = i < rLimit ? aCount + p->ResultPrimOffset : 0; + MatrixKernel* source = s + rCount; + MatrixKernel* ans = a + aCount; + long rc = p->RightSecOffset * p->RightSecStride, + ac = p->ResultSecOffset * p->ResultSecStride, + sCount = p->SecCount; + long rl = sCount - p->RightSecOffset, + al = sCount - p->ResultSecOffset; + for (int j = 0; j < sCount; j++) + { + rc = j < rl ? rc + p->RightSecOffset : 0; + ac = j < al ? ac + p->ResultSecOffset : 0; + op(source + rc, ans + ac); + } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void ComputeRow( - MatrixKernel* left, - MatrixKernel* right, - MatrixKernel* ans, - ScalarDataPacket* scalar, - IntPtr computeHandle, - MatrixParallelTaskPackage* task) + internal static unsafe void Compute_UnaryScalar(MatrixParallelTaskPackage* p) { - long l = task->LeftEnum.secStride, r = task->RightEnum.secStride, a = task->ResultEnum.secStride, li = 0, ri = 0, ai = 0; - for (int i = 0; i < task->secCount; i++) { - ((delegate*)computeHandle)(left + li, right + ri, ans + ai, scalar); + MatrixKernel* s = p->Right, a = p->Result; + long rCount = p->RightPrimOffset * p->RightPrimStride, + aCount = p->ResultPrimOffset * p->ResultPrimStride, + pCount = p->PrimCount; + long rLimit = pCount - p->RightPrimOffset, + aLimit = pCount - p->ResultPrimOffset; + delegate* unmanaged[Cdecl] op = + (delegate* unmanaged[Cdecl])p->ComputeHandle; + ScalarPacket* scalar = p->Scalar; + for (long i = 0; i < pCount; i++) + { + rCount = i < rLimit ? rCount + p->RightPrimOffset : 0; + aCount = i < rLimit ? aCount + p->ResultPrimOffset : 0; + MatrixKernel* source = s + rCount; + MatrixKernel* ans = a + aCount; + long rc = p->RightSecOffset * p->RightSecStride, + ac = p->ResultSecOffset * p->ResultSecStride, + sCount = p->SecCount; + long rl = sCount - p->RightSecOffset, + al = sCount - p->ResultSecOffset; + for (int j = 0; j < sCount; j++) + { + rc = j < rl ? rc + p->RightSecOffset : 0; + ac = j < al ? ac + p->ResultSecOffset : 0; + op(source + rc, ans + ac, scalar); + } } } diff --git a/StgSharp.Common/Mathematics/Linear/MatrixParallel.Task.cs b/StgSharp.Common/Mathematics/Linear/MatrixParallel.Task.cs index 50b43cfe..0952c837 100644 --- a/StgSharp.Common/Mathematics/Linear/MatrixParallel.Task.cs +++ b/StgSharp.Common/Mathematics/Linear/MatrixParallel.Task.cs @@ -37,69 +37,95 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Runtime.InteropServices; +using System.Text.Json.Serialization; using System.Threading; namespace StgSharp.Mathematics { - [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 8)] - public unsafe struct MatrixParallelEnumeration + public unsafe struct ScalarPacket { - [FieldOffset(00)] public MatrixKernel* kernel; + private fixed ulong data[8]; - [FieldOffset(08)] public int begin; - [FieldOffset(12)] public int end; - [FieldOffset(16)] public int offset; - [FieldOffset(20)] public int primStride; - [FieldOffset(24)] public int secStride; - - } - - [StructLayout(LayoutKind.Explicit, Size = 128, Pack = 8)] - public unsafe struct MatrixParallelTaskPackage - { - - [FieldOffset(0)] public MatrixKernel* Left; - [FieldOffset(24)] public MatrixKernel* Result; - [FieldOffset(48)] public MatrixKernel* Right; - [FieldOffset(72)] public ScalarDataPacket* scalerPacket; - [FieldOffset(80)] public int ComputeMode; - [FieldOffset(84)] public int primCount; - [FieldOffset(88)] public int secCount; - [FieldOffset(108)] public IntPtr executionHandle; - - [FieldOffset(0)] public MatrixParallelEnumeration LeftEnum; - [FieldOffset(24)] public MatrixParallelEnumeration ResultEnum; - [FieldOffset(48)] public MatrixParallelEnumeration RightEnum; - - } - - /// - /// Scalar data packet for storing up to 8 parameters as type-erased ulong values Supports - /// reference counting for shared usage across multiple tasks Used to separate scalar data - /// allocation from task allocation and track batch completion - /// - [StructLayout(LayoutKind.Explicit, Size = 72)] - public unsafe struct ScalarDataPacket - { - - public const int MaxScalarCount = 8; - public const int PacketSize = 64; - - [FieldOffset(0)] private fixed ulong Value[8]; - - public ScalarDataPacket() + public ref T Data(int index) where T: unmanaged,INumber { - Unsafe.SkipInit(out this); - } - - public ref T GetScalerRef(int index) where T: unmanaged, INumber - { - if (index is < 0 or >= MaxScalarCount) { - throw new IndexOutOfRangeException("ScalarDataPacket index out of range"); + if (index is < 0 or >= 8) { + throw new IndexOutOfRangeException(); } - return ref Unsafe.As(ref Value[index]); + + return ref Unsafe.As(ref data[index]); } } + + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct MatrixParallelTaskPackage + { + + #region source + + // Group 1: Left/Right matrix kernel base address + public MatrixKernel* Left; // 8 + public MatrixKernel* Right; // 8 + + #endregion + + #region ans + + // Group 2: Result kernel base address + reserved + public MatrixKernel* Result; // 8 + public void* ReservedPtr0; // 8 (extension/reserved) + + #endregion + + #region left enum + + public int LeftPrimOffset; // 4 + public int LeftPrimStride; // 4 + public int LeftSecOffset; // 4 + public int LeftSecStride; // 4 + + #endregion + + #region right enum + + public int RightPrimOffset; // 4 + public int RightPrimStride; // 4 + public int RightSecOffset; // 4 + public int RightSecStride; // 4 + + #endregion + + #region ans enum + + public int ResultPrimOffset; // 4 + public int ResultPrimStride; // 4 + public int ResultSecOffset; // 4 + public int ResultSecStride; // 4 + + #endregion + + #region scalar/compute + + public ScalarPacket* Scalar; // 8 + public IntPtr ComputeHandle; // 8 + + #endregion + + #region global profile + + public int PrimCount; // 4 + public int ComputeMode; // 4 + public int SecCount; // 4 + private readonly int ReservedInt0; // 4 + + #endregion + + #region reserve + + public long ReservedLong0; // 8 + public long ReservedLong1; // 8 + + #endregion + } } diff --git a/StgSharp.Common/Mathematics/Linear/MatrixParallel.TaskWrapFactory.cs b/StgSharp.Common/Mathematics/Linear/MatrixParallel.TaskWrapFactory.cs index 441b0b4a..84922aaa 100644 --- a/StgSharp.Common/Mathematics/Linear/MatrixParallel.TaskWrapFactory.cs +++ b/StgSharp.Common/Mathematics/Linear/MatrixParallel.TaskWrapFactory.cs @@ -40,13 +40,13 @@ namespace StgSharp.Mathematics internal static unsafe class MatrixParallelFactory { - private static readonly SlabAllocator _scalarAllocator = SlabAllocator.Create(64, SlabBufferLayout.Chunked, true); + private static readonly SlabAllocator _scalarAllocator = SlabAllocator.Create(64, SlabBufferLayout.Chunked, true); private static readonly SlabAllocator _wrapAllocator = SlabAllocator.Create(64, SlabBufferLayout.Chunked, true); - public static ScalarDataPacket* CreateScalarPacket() + public static ScalarPacket* CreateScalarPacket() { - ScalarDataPacket* p = (ScalarDataPacket*)_scalarAllocator.Allocate(); + ScalarPacket* p = (ScalarPacket*)_scalarAllocator.Allocate(); return p; } @@ -60,7 +60,7 @@ namespace StgSharp.Mathematics return p; } - public static void Release(ScalarDataPacket* source) + public static void Release(ScalarPacket* source) { _scalarAllocator.Free((nuint)source); } diff --git a/StgSharp.Common/Mathematics/Linear/MatrixParallel.cs b/StgSharp.Common/Mathematics/Linear/MatrixParallel.cs index 942035ad..89c6535a 100644 --- a/StgSharp.Common/Mathematics/Linear/MatrixParallel.cs +++ b/StgSharp.Common/Mathematics/Linear/MatrixParallel.cs @@ -76,12 +76,9 @@ namespace StgSharp.Mathematics private static readonly ManualResetEventSlim _restartEvent = new(false); private static readonly ManualResetEventSlim _startEvent = new(false); - private static MatrixParallelTaskPackage _totalTask; private static readonly object _lock = new(); private static ThreadPriority _AfterWork = ThreadPriority.Lowest; - public static ref MatrixParallelTaskPackage TotalTask => ref _totalTask; - public static bool SupportParallel { get; } = Environment.ProcessorCount > 6; public static bool IsParallelMeaningful { get; } = Environment.ProcessorCount > 6; @@ -139,7 +136,6 @@ namespace StgSharp.Mathematics private Queue[] _cache; private Thread[] _threads; - private MatrixParallelTaskPackage _wrapTask; public MatrixParallelWrap(int id, int wrapSize, bool isFirst) { @@ -169,8 +165,6 @@ namespace StgSharp.Mathematics _threads[end].Start(new Handle(end, this)); } - public ref MatrixParallelTaskPackage WrapTask => ref _wrapTask; - public int ID { get; init; } public Span EnqueuePackage(Span packageSpan) @@ -183,9 +177,9 @@ namespace StgSharp.Mathematics } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetNextWrapTask(int idInWrap, out IntPtr handle) + public Queue GetWorkerTaskQueue(int idInWrap) { - return _cache[idInWrap].TryDequeue(out handle); + return _cache[idInWrap]; } public record class Handle(int InWrapIndex, MatrixParallelWrap Wrap); @@ -203,42 +197,35 @@ namespace StgSharp.Mathematics while (true) { self.Priority = ThreadPriority.Highest; - while (wrap.TryGetNextWrapTask(inWrapIndex, out IntPtr ptr)) { - ExecuteTaskPackage((MatrixParallelTaskPackage*)ptr); - } + ExecuteTaskPackage(wrap.GetWorkerTaskQueue(inWrapIndex)); _ = Interlocked.Decrement(ref RemainThreadCount); self.Priority = _AfterWork; _restartEvent.Wait(); } } - private static unsafe void ExecuteTaskPackage(MatrixParallelTaskPackage* p) + private static unsafe void ExecuteTaskPackage(Queue taskQueue) { - MatrixKernel* left = p->Left, right = p->Left, result = p->Result; - ref MatrixParallelEnumeration leftParam = ref p->LeftEnum, rightParam = ref p->RightEnum, resultParam = ref p->ResultEnum; - int count = p->count; - ScalarDataPacket* scalar = p->scalerPacket; - switch (p->ComputeMode) + while (taskQueue.TryDequeue(out nint handle)) { - case 1: // Unary operation - for (int i = 0; i < length; i++) { } - break; - case 2: - break; - case 3: - break; - case 4: - break; - case -1: - break; - case -2: - break; - case -3: - break; - case -4: - break; - default: - break; + MatrixParallelTaskPackage* p = (MatrixParallelTaskPackage*)handle; + switch (p->ComputeMode) + { + case 1: + Compute_Unary(p); + break; + case 2: + Compute_UnaryScalar(p); + break; + case 3: + Compute_Binary(p); + break; + case 4: + Compute_BinaryScalar(p); + break; + default: + break; + } } } @@ -254,7 +241,7 @@ namespace StgSharp.Mathematics while (true) { self.Priority = ThreadPriority.Highest; - while (wrap.TryGetNextWrapTask(inWrapIndex, out IntPtr ptr)) { + while (wrap.GetWorkerTaskQueue(inWrapIndex, out IntPtr ptr)) { ExecuteTaskPackage((MatrixParallelTaskPackage*)ptr); } _ = Interlocked.Decrement(ref RemainThreadCount); diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index feccfe70..b778176b 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -23,7 +23,7 @@ (LAST(reset2) << 2) | (LAST(reset1) << 1) | LAST(reset0) #define SSE_INSERT_SIMPLE(source, target) SSE_INSERT(source, target, 0, 0, 0, 0) #define M256_PERMUTE(s3, s2, s1, s0) (LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0) - +#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index) #define zero_vec _mm_setzero_ps() #define zero_vec_256 _mm256_setzero_ps() @@ -58,6 +58,10 @@ typedef union column_4 { float m[4][4]; } __4_columnset; +typedef union scalar_pack { + uint64_t data[8]; +} scalar_pack; + typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target); typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans); typedef void(SN_DECL *MATRIXTRANSPOSEPROC)(void *source, void *target);