From d62e59cb933c0e98d5db64b8c94e3e5127cf07c9 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 24 Feb 2026 15:12:23 +0800 Subject: [PATCH] Switch matrix parallel to task stealing, type and wrap are refacored, thread execution core has not yet --- .../Numeric/MatrixCompute.Float.cs | 32 +- .../Numeric/MatrixCompute.double.cs | 38 +- .../Numeric/MatrixParallel.Publish.cs | 56 +-- .../Numeric/MatrixParallel.cs | 23 +- ...lHandle.cs => MatrixParallelHandle.cs.old} | 394 +++++++++--------- .../Numeric/MatrixParallelQueue.cs | 81 ++++ ...Model.cs => MatrixParallelThread.Model.cs} | 268 ++++++------ .../Numeric/MatrixParallelThread.cs | 323 ++------------ .../Numeric/MatrixParallelWrap.cs | 226 ++++------ .../Numeric/MatrixParallelWrap.cs.old | 189 +++++++++ .../Numeric/MatrixTaskFactory.cs | 34 +- StgSharp.Native/SIMDID.x86.md | 95 ++++- StgSharp.Native/StgSharp.Native.vcxproj | 8 +- .../StgSharp.Native.vcxproj.filters | 6 + StgSharp.Native/sn-head/StgSharpNative.h | 7 +- .../sn-head/sn_intrincontext.x86.h | 55 ++- StgSharp.Native/sn-head/sn_target.h | 2 - StgSharp.Native/sn-src/cpuid.x86.c | 54 ++- StgSharp.Native/sn-src/load_intrinsic.x86.c | 17 +- StgSharp.Native/sn-src/m.fma_tile.x86.c | 26 -- StgSharp.Native/sn-src/t.affinity.c | 31 ++ 21 files changed, 1000 insertions(+), 965 deletions(-) rename StgSharp.Mathematics/Numeric/{MatrixParallelHandle.cs => MatrixParallelHandle.cs.old} (97%) create mode 100644 StgSharp.Mathematics/Numeric/MatrixParallelQueue.cs rename StgSharp.Mathematics/Numeric/{MatrixComputeModel.cs => MatrixParallelThread.Model.cs} (95%) create mode 100644 StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs.old create mode 100644 StgSharp.Native/sn-src/t.affinity.c diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs index 06cf9b82..9e676bea 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.Float.cs @@ -76,7 +76,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -85,9 +85,9 @@ namespace StgSharp.Mathematics.Numeric package->SecCount = ans.KernelRowLength; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f32].buffer_add; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } @@ -112,7 +112,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Answer = (nint)ans.Buffer; package->ElementSize = sizeof(float); package->PrimCount = ans.KernelColumnLength; @@ -120,8 +120,8 @@ namespace StgSharp.Mathematics.Numeric package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.ANS_SCALAR_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f32].buffer_fill; package->Scalar = scalar; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); + taskGroup.LaunchParallel(SleepMode.DeepSleep); MatrixParallelFactory.Release(scalar); taskGroup.Dispose(); } @@ -178,7 +178,7 @@ namespace StgSharp.Mathematics.Numeric } return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -188,10 +188,10 @@ namespace StgSharp.Mathematics.Numeric package->PrimCount = ans.KernelColumnLength; package->SecCount = ans.KernelRowLength; package->ElementSize = (int)MatrixElementType.F32; - package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA); - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.SPECIAL, MatrixOperationSpecial.SEQ_FMA); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } @@ -231,7 +231,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Right = (nint)left.Buffer; package->Answer = (nint)ans.Buffer; package->ElementSize = sizeof(float); @@ -240,9 +240,9 @@ namespace StgSharp.Mathematics.Numeric package->Scalar = scalar; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f32].buffer_scalar_mul; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); MatrixParallelFactory.Release(scalar); return; @@ -287,7 +287,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -296,9 +296,9 @@ namespace StgSharp.Mathematics.Numeric package->SecCount = left.KernelRowLength; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f32].buffer_sub; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } diff --git a/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs b/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs index fd21950d..7e6ce90f 100644 --- a/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs +++ b/StgSharp.Mathematics/Numeric/MatrixCompute.double.cs @@ -72,7 +72,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -81,9 +81,9 @@ namespace StgSharp.Mathematics.Numeric package->SecCount = ans.KernelRowLength; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f64].buffer_add; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } @@ -108,7 +108,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Answer = (nint)ans.Buffer; package->ElementSize = sizeof(double); package->PrimCount = ans.KernelColumnLength; @@ -116,8 +116,8 @@ namespace StgSharp.Mathematics.Numeric package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.ANS_SCALAR_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f64].buffer_fill; package->Scalar = scalar; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); + taskGroup.LaunchParallel(SleepMode.DeepSleep); MatrixParallelFactory.Release(scalar); taskGroup.Dispose(); } @@ -139,8 +139,6 @@ namespace StgSharp.Mathematics.Numeric throw new IndexOutOfRangeException("Matrix dimensions must agree."); } long count = (long)left.KernelColumnLength * left.KernelRowLength; - - // int size = MatrixPanel.KernelSideCount(); if (count <= 1024) { MatrixParallelThread[] pool = null; @@ -162,18 +160,14 @@ namespace StgSharp.Mathematics.Numeric (MatrixKernel*)right.Buffer, (MatrixKernel*)ans.Buffer, enumeration); - - // Console.WriteLine($"{i},{j},{ans[4*i,4*j]:0.0E00}\t"); } - - // Console.WriteLine(); } if (count > 256) { MatrixParallel.ReturnParallelResources(ref pool!); } return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -183,10 +177,10 @@ namespace StgSharp.Mathematics.Numeric package->PrimCount = ans.KernelColumnLength; package->SecCount = ans.KernelRowLength; package->ElementSize = (int)MatrixElementType.F64; - package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationSpecial.SEQ_FMA); - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.SPECIAL, MatrixOperationSpecial.SEQ_FMA); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } @@ -226,7 +220,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Right = (nint)left.Buffer; package->Answer = (nint)ans.Buffer; package->ElementSize = sizeof(double); @@ -235,9 +229,9 @@ namespace StgSharp.Mathematics.Numeric package->Scalar = scalar; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f64].buffer_scalar_mul; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); MatrixParallelFactory.Release(scalar); return; @@ -282,7 +276,7 @@ namespace StgSharp.Mathematics.Numeric MatrixParallel.ReturnParallelResources(ref pool); return; } - MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); + MatrixParallelTaskPackage* package = MatrixParallelFactory.CreateBaseTask(); package->Left = (nint)left.Buffer; package->Right = (nint)right.Buffer; package->Answer = (nint)ans.Buffer; @@ -291,9 +285,9 @@ namespace StgSharp.Mathematics.Numeric package->SecCount = left.KernelRowLength; package->ComputeMode = new MatrixOpMode(MatrixIndexStyle.BUFFER_INDEX, MatrixOperationParam.LEFT_RIGHT_ANS_PARAM); package->ComputeHandle = (IntPtr)NativeIntrinsic.Context.mat[f64].buffer_sub; - MatrixParallelHandle taskGroup = MatrixParallel.ScheduleTask(package); + MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask(package); - MatrixParallel.LaunchParallel(taskGroup, SleepMode.DeepSleep); + taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.Dispose(); return; } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs index 0383cd06..2ac93bfe 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs @@ -46,20 +46,22 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe MatrixParallelHandle ScheduleTask(MatrixParallelTaskPackage* package) + internal static unsafe MatrixParallelWrap ScheduleTask( + MatrixParallelTaskPackage* package + ) { - MatrixParallelTaskPackage* nonGeneric = package; - switch (nonGeneric->ComputeMode.OperationStyle) + switch (package->ComputeMode.OperationStyle) { case MatrixIndexStyle.BUFFER_INDEX: - return ScheduleBufferTask(nonGeneric); + return ScheduleBufferTask(package); default: - throw new InvalidOperationException($"Unknown mode {nonGeneric->ComputeMode.OperationStyle}."); + throw new InvalidOperationException($"Unknown mode {package->ComputeMode.OperationStyle}."); } } - private static unsafe MatrixParallelHandle ScheduleBufferTask( - MatrixParallelTaskPackage* package) + private static unsafe MatrixParallelWrap ScheduleBufferTask( + MatrixParallelTaskPackage* package + ) { int primCount = package->PrimCount; int secCount = package->SecCount; @@ -69,44 +71,10 @@ namespace StgSharp.Mathematics.Numeric long recommendCount = count / 1024; MatrixParallelThread[] pool = LeadParallel((int)recommendCount); - MatrixParallelHandle handle = new MatrixParallelHandle(pool); - if (handle.Wraps.Length == 1) - { - // Console.WriteLine($"Scheduling into {pool.Length} threads."); - ref long offsetRef = ref Unsafe.As(ref package->PrimTileOffset); - offsetRef = 0; - ref long lengthRef = ref Unsafe.As(ref package->PrimCount); - lengthRef = count; - handle.Wraps[0].WrapTask.Enqueue((nint)package); + MatrixParallelWrap handle = new MatrixParallelWrap(pool); - // Console.WriteLine($"Scheduling {count} kernels into thread"); - return handle; - } - long actual = count / pool.Length; - long extra = count % pool.Length; - - long offset = 0; - MatrixParallelTaskPackage* current = package; - - // Console.WriteLine($"Scheduling into {pool.Length} threads."); - foreach (MatrixParallelWrap wrap in handle.Wraps) - { - long wrapTaskLength = actual * wrap.WrapTaskCapacity; - long ex = Math.Min(extra, wrap.WrapTaskCapacity); - long length = wrapTaskLength + ex; - extra -= ex; - ref long offsetRef = ref Unsafe.As(ref current->PrimTileOffset); - offsetRef = offset; - ref long lengthRef = ref Unsafe.As(ref current->PrimCount); - lengthRef = length; - wrap.WrapTask.Enqueue((nint) current); - - // Console.WriteLine($"Offset is {offset}"); - // Console.WriteLine($"Scheduling {length} kernels into {wrap.WrapTaskCapacity} threads"); - // Console.WriteLine(); - current = MatrixParallelFactory.FromExistPackage(current); - offset += length; - } + // TODO set parallel queue for handle + handle.TaskQueue = MatrixParallelQueue.Rent(package); return handle; } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallel.cs b/StgSharp.Mathematics/Numeric/MatrixParallel.cs index 7e06a08e..ae5b6fd8 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallel.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="MatrixParallel" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -49,7 +49,7 @@ namespace StgSharp.Mathematics.Numeric #endif _lock = new(); - private static ThreadPriority _AfterWork = ThreadPriority.Lowest; + // private static ThreadPriority _AfterWork = ThreadPriority.Lowest; public static bool SupportParallel { get; } = Environment.ProcessorCount > 6; @@ -64,17 +64,6 @@ namespace StgSharp.Mathematics.Numeric Monitor.Exit(_lock); } - public static void LaunchParallel(MatrixParallelHandle handle, SleepMode afterWork) - { - _AfterWork = (ThreadPriority)afterWork; - MatrixParallelThread leader = handle.GetLeader(); - foreach (MatrixParallelWrap w in handle.Wraps) { - w.SetScheduler(); - } - leader.MatrixParallelLeader(); - handle.WaitAllWraps(); - } - internal static void Init() { if (IsInitialized) { @@ -88,8 +77,10 @@ namespace StgSharp.Mathematics.Numeric } int count = Environment.ProcessorCount - 4; _threads = new(count); - for (int i = 0; i < count; i++) { - _threads.Push(new()); + for (int i = 0; i < count; i++) + { + MatrixParallelThread thread = new MatrixParallelThread(); + _threads.Push(thread); } _awaitingLeaderHandle = new AwaitingQueue(); IsInitialized = true; diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs b/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs.old similarity index 97% rename from StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs rename to StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs.old index b5325bff..3ab1a077 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelHandle.cs.old @@ -1,197 +1,197 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="MatrixParallelHandle" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using StgSharp.Collections; -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; - -namespace StgSharp.Mathematics.Numeric -{ - public class MatrixParallelHandle : IDisposable - { - - private static readonly (int a, int b)[] LookupTable = [ - (1, 1), // 7 - (0, 2), // 8 (no non-zero solution) - (3, 0), // 9 (no non-zero solution) - (2, 1), // 10 - (1, 2), // 11 - (0, 3), // 12 (no non-zero solution) - (3, 1), // 13 - (2, 2), // 14 - (1, 3), // 15 - (4, 1), // 16 - (3, 2), // 17 - (2, 3), // 18 - (1, 4), // 19 - (4, 2), // 20 - ]; - private MatrixParallelThread[] _threads; - private MatrixParallelWrap[] _wraps; - private bool _disposed = false; - private int _count; - private ManualResetEventSlim _leaderEvent = new ManualResetEventSlim(false); - - internal MatrixParallelHandle(MatrixParallelThread[] threadArray) - { - _threads = threadArray; - IsMultithread = threadArray.Length > 1; - SliceToWraps(); - } - - public ReadOnlySpan Threads - { - get - { - if (_disposed) { - throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); - } - return _threads; - } - } - - internal ReadOnlySpan Wraps - { - get - { - if (_disposed) { - throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); - } - return _wraps; - } - } - - private bool IsMultithread { get; init; } - - public void Dispose() - { - if (_disposed) { - return; - } - MatrixParallel.ReturnParallelResources(ref _threads); - _disposed = true; - } - - public MatrixParallelThread GetLeader() - { - MatrixParallelThread leader = Threads[1]; - leader.Role = MatrixParallelThread.RoleType.Leader; - leader.LeaderEvent = _leaderEvent; - return leader; - } - - public void WaitAllWraps() - { - if (_disposed) { - throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); - } - SpinWait.SpinUntil(() => Volatile.Read(ref _count) == 0); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ReportWrapAccomplished() - { - if (_disposed) { - throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); - } - Interlocked.Decrement(ref _count); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static (int count_3, int count_4) FastDecompose(int c) - { - /* - * Thread wrapping to accelerate matrix operations by dividing threads into groups (pool). - * For M pool, each containing an average of N threads, we satisfy the condition M * N = T - 4, - * where T is the total thread _count of the CPU. Typically, for modern consumer CPUs, T is around 16. - * For simplicity, let M * N = 16. - * - * The total time complexity for publishing tasks is O(M + N): - * - O(M) accounts for distributing tasks across M pool. - * - O(N) accounts for threads within each wrap fetching tasks from the wrap queue. - * - * Using the inequality Sqrt(M * N) <= (M + N) / 2 (derived from the arithmetic-geometric mean inequality), - * the optimal configuration occurs when M = N. For M * N = 16, this gives M = N = 4. - * - * To ensure compatibility across a wide range of CPU architectures, - * we allow each wrap to contain either 3 or 4 threads, providing flexibility while maintaining efficiency. - */ - if (c < 7) - { - return (0, 0); - } - - int t = c - 7; - int k = t / 14; - int d = t % 14; - - (int a0, int b0) = LookupTable[d]; - - return (a0 + (2 * k), b0 + (2 * k)); - } - - private void SliceToWraps() - { - if (_disposed) { - throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); - } - if (_wraps is not null) { - return; - } - int count = _threads.Length; - if (count < 6) - { - MatrixParallelWrap wrap = new MatrixParallelWrap(this, 0, count); - _wraps = [ wrap ]; - } else - { - (int c3, int c4) = FastDecompose(_threads.Length); - _wraps = new MatrixParallelWrap[c3 + c4]; - int begin = 0; - int i = 0; - for (; i < c3; i++) - { - _wraps[i] = new MatrixParallelWrap(this, begin, 3); - begin += 3; - } - for (; i < c3 + c4; i++) - { - _wraps[i] = new MatrixParallelWrap(this, begin, 4); - begin += 4; - } - } - _count = _wraps.Length; - } - - } -} +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixParallelHandle" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.Collections; +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public class MatrixParallelHandle : IDisposable + { + + private static readonly (int a, int b)[] LookupTable = [ + (1, 1), // 7 + (0, 2), // 8 (no non-zero solution) + (3, 0), // 9 (no non-zero solution) + (2, 1), // 10 + (1, 2), // 11 + (0, 3), // 12 (no non-zero solution) + (3, 1), // 13 + (2, 2), // 14 + (1, 3), // 15 + (4, 1), // 16 + (3, 2), // 17 + (2, 3), // 18 + (1, 4), // 19 + (4, 2), // 20 + ]; + private MatrixParallelThread[] _threads; + private MatrixParallelWrap[] _wraps; + private bool _disposed = false; + private int _count; + private ManualResetEventSlim _leaderEvent = new ManualResetEventSlim(false); + + internal MatrixParallelHandle(MatrixParallelThread[] threadArray) + { + _threads = threadArray; + IsMultithread = threadArray.Length > 1; + SliceToWraps(); + } + + public ReadOnlySpan Threads + { + get + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); + } + return _threads; + } + } + + internal ReadOnlySpan Wraps + { + get + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); + } + return _wraps; + } + } + + private bool IsMultithread { get; init; } + + public void Dispose() + { + if (_disposed) { + return; + } + MatrixParallel.ReturnParallelResources(ref _threads); + _disposed = true; + } + + public MatrixParallelThread GetLeader() + { + MatrixParallelThread leader = Threads[1]; + leader.Role = MatrixParallelThread.RoleType.Leader; + leader.LeaderEvent = _leaderEvent; + return leader; + } + + public void WaitAllWraps() + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); + } + SpinWait.SpinUntil(() => Volatile.Read(ref _count) == 0); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ReportWrapAccomplished() + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); + } + Interlocked.Decrement(ref _count); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static (int count_3, int count_4) FastDecompose(int c) + { + /* + * Thread wrapping to accelerate matrix operations by dividing threads into groups (pool). + * For M pool, each containing an average of N threads, we satisfy the condition M * N = T - 4, + * where T is the total thread _count of the CPU. Typically, for modern consumer CPUs, T is around 16. + * For simplicity, let M * N = 16. + * + * The total time complexity for publishing tasks is O(M + N): + * - O(M) accounts for distributing tasks across M pool. + * - O(N) accounts for threads within each wrap fetching tasks from the wrap queue. + * + * Using the inequality Sqrt(M * N) <= (M + N) / 2 (derived from the arithmetic-geometric mean inequality), + * the optimal configuration occurs when M = N. For M * N = 16, this gives M = N = 4. + * + * To ensure compatibility across a wide range of CPU architectures, + * we allow each wrap to contain either 3 or 4 threads, providing flexibility while maintaining efficiency. + */ + if (c < 7) + { + return (0, 0); + } + + int t = c - 7; + int k = t / 14; + int d = t % 14; + + (int a0, int b0) = LookupTable[d]; + + return (a0 + (2 * k), b0 + (2 * k)); + } + + private void SliceToWraps() + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelHandle).Name); + } + if (_wraps is not null) { + return; + } + int count = _threads.Length; + if (count < 6) + { + MatrixParallelWrap wrap = new MatrixParallelWrap(this, 0, count); + _wraps = [ wrap ]; + } else + { + (int c3, int c4) = FastDecompose(_threads.Length); + _wraps = new MatrixParallelWrap[c3 + c4]; + int begin = 0; + int i = 0; + for (; i < c3; i++) + { + _wraps[i] = new MatrixParallelWrap(this, begin, 3); + begin += 3; + } + for (; i < c3 + c4; i++) + { + _wraps[i] = new MatrixParallelWrap(this, begin, 4); + begin += 4; + } + } + _count = _wraps.Length; + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelQueue.cs b/StgSharp.Mathematics/Numeric/MatrixParallelQueue.cs new file mode 100644 index 00000000..5d67062d --- /dev/null +++ b/StgSharp.Mathematics/Numeric/MatrixParallelQueue.cs @@ -0,0 +1,81 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixParallelQueue" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using Microsoft.VisualBasic.FileIO; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public unsafe class MatrixParallelQueue + { + + private static ConcurrentStack _recycle = []; + private MatrixParallelTaskPackage* _package; + private ulong _cursor; + private ulong _maxCount; + + private MatrixParallelQueue() { } + + public MatrixParallelTaskPackage* Package => _package; + + public static MatrixParallelQueue Rent( + MatrixParallelTaskPackage* pack + ) + { + if (!_recycle.TryPop(out MatrixParallelQueue? queue)) { + queue = new MatrixParallelQueue(); + } + queue._package = pack; + queue._maxCount = (ulong)pack->PrimCount * (ulong)pack->SecCount; + queue._cursor = 0; + return queue; + } + + public static void Return( + MatrixParallelQueue queue + ) + { + _recycle.Push(queue); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (ulong count, ulong offset) Steal( + ulong request + ) + { + ulong cursor = Interlocked.Add(ref _cursor, request) - request; + return (Math.Min(request, Math.Max(0, _maxCount - cursor)), cursor); + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs similarity index 95% rename from StgSharp.Mathematics/Numeric/MatrixComputeModel.cs rename to StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs index 29323d38..62908fcc 100644 --- a/StgSharp.Mathematics/Numeric/MatrixComputeModel.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.Model.cs @@ -1,134 +1,134 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="MatrixComputeModel" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using StgSharp.HighPerformance; -using System; -using System.Runtime.CompilerServices; - -namespace StgSharp.Mathematics.Numeric -{ - internal static class MatrixComputeModel - { - - internal static unsafe void BufferComputeBinary( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), - (void*)(p->Right + offset), - (void*)(p->Answer + offset), - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeBinaryScalar( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ScalarPacket* scalar = p->Scalar; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), - (void*)(p->Right + offset), - (void*)(p->Answer + offset), - scalar, - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeNoOperatorScalar( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Answer + offset), - p->Scalar, count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeUnary( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Answer + offset), - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void BufferComputeUnaryScalar( - MatrixParallelTaskPackage* p - ) - { - int size = p->ElementSize * 16; - long count = Unsafe.As(ref p->PrimCount); - long offset = Unsafe.As(ref p->PrimTileOffset) * size; - ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), - (void*)(p->Answer + offset), - p->Scalar, - count); - MatrixParallelFactory.Release(p); - } - - internal static unsafe void SpecialBufferCompute( - MatrixParallelTaskPackage* p - ) - { - switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle) - { - case MatrixOperationSpecial.SEQ_FMA: - switch ((MatrixElementType)p->ElementSize) - { - case MatrixElementType.F64: - MatrixCompute.ComputeFmaPanelF64(p); - return; - case MatrixElementType.F32: - MatrixCompute.ComputeFmaPanelF32(p); - break; - default: - break; - } - break; - default: - break; - } - MatrixParallelFactory.Release(p); - } - - } -} +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixParallelThread.Model" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.HighPerformance; +using System; +using System.Runtime.CompilerServices; + +namespace StgSharp.Mathematics.Numeric +{ + internal static class MatrixComputeModel + { + + internal static unsafe void BufferComputeBinary( + MatrixParallelTaskPackage* p + ) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), + (void*)(p->Right + offset), + (void*)(p->Answer + offset), + count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void BufferComputeBinaryScalar( + MatrixParallelTaskPackage* p + ) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + ScalarPacket* scalar = p->Scalar; + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Left + offset), + (void*)(p->Right + offset), + (void*)(p->Answer + offset), + scalar, + count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void BufferComputeNoOperatorScalar( + MatrixParallelTaskPackage* p + ) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Answer + offset), + p->Scalar, count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void BufferComputeUnary( + MatrixParallelTaskPackage* p + ) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), + (void*)(p->Answer + offset), + count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void BufferComputeUnaryScalar( + MatrixParallelTaskPackage* p + ) + { + int size = p->ElementSize * 16; + long count = Unsafe.As(ref p->PrimCount); + long offset = Unsafe.As(ref p->PrimTileOffset) * size; + ((delegate* unmanaged[Cdecl])p->ComputeHandle)((void*)(p->Right + offset), + (void*)(p->Answer + offset), + p->Scalar, + count); + MatrixParallelFactory.Release(p); + } + + internal static unsafe void SpecialBufferCompute( + MatrixParallelTaskPackage* p + ) + { + switch ((MatrixOperationSpecial)p->ComputeMode.ParameterStyle) + { + case MatrixOperationSpecial.SEQ_FMA: + switch ((MatrixElementType)p->ElementSize) + { + case MatrixElementType.F64: + MatrixCompute.ComputeFmaPanelF64(p); + return; + case MatrixElementType.F32: + MatrixCompute.ComputeFmaPanelF32(p); + break; + default: + break; + } + break; + default: + break; + } + MatrixParallelFactory.Release(p); + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs index f26f3764..019bc438 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelThread.cs @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Internal; using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; @@ -37,42 +38,42 @@ using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public partial class MatrixParallelThread : IDisposable + public unsafe partial class MatrixParallelThread : IDisposable { - // Affinity binding info for this thread - private AffinityBinding _affinity; - private bool _affinityApplied; - + private IntPtr* _context; private bool disposedValue; private readonly ManualResetEventSlim _resetEvent; private readonly Thread _managedThread; public MatrixParallelThread() { - // Allocate a unique logical processor for this worker (mutually exclusive) - _affinity = AffinityAllocator.Allocate(); _resetEvent = new ManualResetEventSlim(false); _managedThread = new Thread(MatrixParallelWorker); _managedThread.Start(); } - // Global runtime switch: true => UMA/no-affinity; false => NUMA/affinity (auto-detected) - public static bool UseUmaAffinityPath { get; } = DetectUmaAffinityPath(); - // ~MatrixComputeThread() // { // Dispose(disposing: false); // } - public ThreadPriority AfterWork - { - get => field; - set => field = value; - } + public ThreadPriority AfterWork { get; set; } internal ManualResetEventSlim? LeaderEvent { get; set; } + [DllImport( + dllName:Native.LibName, + EntryPoint = "sn_set_thread_affinity", + CallingConvention = CallingConvention.Cdecl)] + internal static extern void SetThreadAffinity( + ulong affinityMask + ); + + #region wrap belong + + internal MatrixParallelWrap BindedWrap { get; set; } + public void Dispose() { Dispose(disposing:true); @@ -81,45 +82,39 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] public static unsafe void ExecuteTaskPackage( - Queue taskQueue + MatrixParallelQueue taskQueue ) { - while (taskQueue.TryDequeue(out nint handle)) + MatrixParallelTaskPackage* p = taskQueue.Package; + switch (p->ComputeMode.OperationStyle) { - MatrixParallelTaskPackage* p = (MatrixParallelTaskPackage*)handle; - if ((short)p->ComputeMode.ParameterStyle < 0) - { - MatrixComputeModel.SpecialBufferCompute(p); - } else - { - switch (p->ComputeMode.OperationStyle) + case MatrixIndexStyle.BUFFER_INDEX: + switch (p->ComputeMode.ParameterStyle) { - case MatrixIndexStyle.BUFFER_INDEX: - switch (p->ComputeMode.ParameterStyle) - { - case MatrixOperationParam.RIGHT_ANS_PARAM: - MatrixComputeModel.BufferComputeUnary(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: - MatrixComputeModel.BufferComputeBinary(p); - break; - case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.BufferComputeUnaryScalar(p); - break; - case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: - MatrixComputeModel.BufferComputeBinaryScalar(p); - break; - case MatrixOperationParam.ANS_SCALAR_PARAM: - MatrixComputeModel.BufferComputeNoOperatorScalar(p); - break; - default: - break; - } + case MatrixOperationParam.RIGHT_ANS_PARAM: + BufferComputeUnary(p); + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_PARAM: + BufferComputeBinary(p); + break; + case MatrixOperationParam.RIGHT_ANS_SCALAR_PARAM: + BufferComputeUnaryScalar(p); + break; + case MatrixOperationParam.LEFT_RIGHT_ANS_SCALAR_PARAM: + BufferComputeBinaryScalar(p); + break; + case MatrixOperationParam.ANS_SCALAR_PARAM: + BufferComputeNoOperatorScalar(p); break; default: break; } - } + break; + case MatrixIndexStyle.SPECIAL: + SpecialBufferCompute(p); + break; + default: + break; } } @@ -143,33 +138,12 @@ namespace StgSharp.Mathematics.Numeric LeaderEvent?.Reset(); // Console.WriteLine("Leader is computing"); -#if WINDOWS - if (!UseUmaAffinityPath) - { - // NUMA/affinity path: bind to this thread's reserved mask, then restore - nuint previousMask = SetThreadAffinityMask(GetCurrentThread(), _affinity.Mask); - bool affinityChanged = previousMask != 0; // nonzero => success - Thread.Sleep(0); - ExecuteTaskPackage(Tasks); - if (affinityChanged) { - _ = SetThreadAffinityMask(GetCurrentThread(), previousMask); - } - } else - { - // UMA path: no affinity binding - ExecuteTaskPackage(Tasks); - } -#else - // Non-Windows: no affinity control - ExecuteTaskPackage(Tasks); -#endif - BindedWrap.ReportThreadAccomplished(); // Console.WriteLine("leader's computing is done"); Role = RoleType.Worker; LeaderEvent = null; - current.Priority = ThreadPriority.Normal; + current.Priority = BindedWrap.AfterWork; } protected virtual void Dispose( @@ -184,51 +158,18 @@ namespace StgSharp.Mathematics.Numeric _managedThread.Join(); } - // Release the reserved logical processor back to the pool - AffinityAllocator.Release(_affinity.LogicalProcessor); - disposedValue = true; } } - private static bool DetectUmaAffinityPath() - { -#if WINDOWS - try - { - // Quick heuristic: single CPU looks like UMA - if (Environment.ProcessorCount <= 1) - { - return true; - } - - // Query NUMA topology: highest node number 0 => single node => UMA - uint highest = 0; - if (GetNumaHighestNodeNumber(out highest)) { - return highest == 0; - } - - // If query fails, prefer UMA (safer: no affinity changes) - return true; - } - catch - { - return true; - } -#else - // Non-Windows: treat as UMA (no affinity API available) - return true; -#endif - } - private unsafe void MatrixParallelWorker() { // Bind OS-thread affinity once when the worker starts running - ApplyAffinityIfNeeded(); while (true) { // Thread self = Thread.CurrentThread; + AfterWork = BindedWrap.AfterWork; _managedThread.Priority = ThreadPriority.Highest; _resetEvent.Wait(); _resetEvent.Reset(); @@ -237,23 +178,11 @@ namespace StgSharp.Mathematics.Numeric case RoleType.Worker://worker // _ = Interlocked.Decrement(ref RemainThreadCount); // Console.WriteLine($"worker {Environment.CurrentManagedThreadId} start"); - ExecuteTaskPackage(Tasks); + ExecuteTaskPackage(BindedWrap.TaskQueue); BindedWrap.ReportThreadAccomplished(); // Console.WriteLine($"worker {Environment.CurrentManagedThreadId} end"); break; - case RoleType.Scheduler://scheduler - // Console.WriteLine($"scheduler {Environment.CurrentManagedThreadId} start"); - BindedWrap.PublishTaskToWorkers(); - BindedWrap.SetAllWorker(); - - // Console.WriteLine($"scheduler {Environment.CurrentManagedThreadId} done"); - ExecuteTaskPackage(Tasks); - BindedWrap.ReportThreadAccomplished(); - BindedWrap.WaitAllWorker(); - BindedWrap.ReportWrapAccomplished(); - Role = RoleType.Worker; - break; case RoleType.Leader://leader, error because leader should be at outer thread but not here break; default: @@ -263,174 +192,12 @@ namespace StgSharp.Mathematics.Numeric } } - #region core affinity - - /* - * data to record core affinity - * preparation for future NUMA affinity setting - */ - - // Represents the binding information for a worker thread - internal readonly struct AffinityBinding - { - - public readonly int LogicalProcessor; - public readonly nuint Mask; - - public AffinityBinding( - int logicalProcessor, - nuint mask - ) - { - LogicalProcessor = logicalProcessor; - Mask = mask; - } - - } - - // Thread-safe allocator that hands out unique logical processors (mutually exclusive) - internal static class AffinityAllocator - { - - // Bitset of reserved logical processors (0..63). One bit per CPU. - private static long s_reservedMask; // treat as unsigned bits - - public static AffinityBinding Allocate() - { - int cpuCount = Environment.ProcessorCount; - if (cpuCount > 64) - { - cpuCount = 64; // SetThreadAffinityMask supports up to 64-bit mask - } - - while (true) - { - long snapshot = Volatile.Read(ref s_reservedMask); - - // Find first zero bit - int candidate = -1; - long bit = 0; - for (int i = 0; i < cpuCount; i++) - { - bit = 1L << i; - if ((snapshot & bit) == 0) - { - candidate = i; - break; - } - } - - if (candidate < 0) - { - // No free CPU found: fall back to CPU 0 (shared) - return new AffinityBinding(0, (nuint)1); - } - - long desired = snapshot | bit; - if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot) - { - // Successfully reserved candidate - nuint mask = (nuint)(1UL << candidate); - return new AffinityBinding(candidate, mask); - } - - // else: another thread raced us; retry - } - } - - public static void Release( - int logicalProcessor - ) - { - if ((uint)logicalProcessor >= 64u) { - return; - } - - long bit = 1L << logicalProcessor; - while (true) - { - long snapshot = Volatile.Read(ref s_reservedMask); - if ((snapshot & bit) == 0) - { - return; // already free - } - - long desired = snapshot & ~bit; - if (Interlocked.CompareExchange(ref s_reservedMask, desired, snapshot) == snapshot) { - return; - } - } - } - - } - - private void ApplyAffinityIfNeeded() - { - if (_affinityApplied) { - return; - } - -#if WINDOWS - try - { - // On Windows pin the current native thread to the reserved logical processor - SetThreadAffinityMask(GetCurrentThread(), _affinity.Mask); - } - catch - { - // Ignore binding errors; continue without affinity - } -#elif LINUX - -#endif - - _affinityApplied = true; - } - - #endregion - - -#if WINDOWS - [DllImport("kernel32.dll")] - private static extern IntPtr GetCurrentThread(); - - [DllImport("kernel32.dll")] - private static extern nuint SetThreadAffinityMask( - IntPtr hThread, - nuint dwThreadAffinityMask - ); - - [LibraryImport("kernel32.dll")] - [return: MarshalAs(UnmanagedType.Bool)] - private static partial bool GetNumaHighestNodeNumber( - out uint HighestNodeNumber - ); - -#elif LINUX - -#endif - - #region wrap belong - - internal MatrixParallelWrap BindedWrap { get; set; } = null!; - - internal Queue Tasks { get; } = []; - - public RoleType Role - { - get; - internal set - { - // Console.WriteLine($"Thread {_managedThread.ManagedThreadId} set to {value}, from thread {Environment.CurrentManagedThreadId}"); - field = value; - } - } + public RoleType Role { get; internal set; } public enum RoleType : int { Worker = 0, - Scheduler = 1, Leader = 2, } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs index 325c8af9..b7acb535 100644 --- a/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs +++ b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs @@ -25,159 +25,18 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- -using StgSharp.HighPerformance; +using StgSharp.Collections; using System; using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Net.WebSockets; using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - public unsafe class MatrixParallelWrap - { - - private int _begin; - - private int _remainThreadCount; - private readonly MatrixParallelHandle _handle; - - internal MatrixParallelWrap( - MatrixParallelHandle handle, - int begin, - int count - ) - { - WrapTaskCapacity = count; - _handle = handle; - _begin = begin; - for (int i = 0; i < WrapTaskCapacity; i++) - { - Threads[i].Role = MatrixParallelThread.RoleType.Worker; - Threads[i].BindedWrap = this; - } - Threads[0].Role = MatrixParallelThread.RoleType.Scheduler; - } - - public static SleepMode AfterWork - { - get; - set => Interlocked.Exchange(ref Unsafe.As(ref field), (int)value); - } - - internal int Magic { get; set; } - - private ReadOnlySpan Threads - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => _handle.Threads[_begin..(_begin + WrapTaskCapacity)]; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ReportThreadAccomplished() - { - Interlocked.Decrement(ref _remainThreadCount); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ReportWrapAccomplished() - { - _handle.ReportWrapAccomplished(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetAllWorker() - { - for (int i = 1; i < WrapTaskCapacity; i++) { - Threads[i].Restart(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void WaitAllWorker() - { - SpinWait.SpinUntil(() => Volatile.Read(ref _remainThreadCount) == 0); - } - - internal void PublishTaskToWorkers() - { - Queue tasks = WrapTask; - while (tasks.TryDequeue(out nint handle)) - { - MatrixParallelTaskPackage* task = (MatrixParallelTaskPackage*)handle; - if ((int)task->ComputeMode.OperationStyle > 0) - { - PublishBufferTaskToWorkers(task); - } else - { - throw new NotImplementedException(); - } - } - } - - internal void SetScheduler() - { - MatrixParallelThread thread = Threads[0]; - thread.Restart(); - _ = Interlocked.Exchange(ref _remainThreadCount, WrapTaskCapacity); - } - - private void PublishBufferTaskToWorkers( - MatrixParallelTaskPackage* package - ) - { - ReadOnlySpan pool = Threads; - if (pool.Length == 1) - { - pool[0].Tasks.Enqueue((nint)package); - return; - } - long count = Unsafe.As(ref package->PrimCount); - long actual = count / pool.Length; - long extra = count % pool.Length; - - long offset = 0; - MatrixParallelTaskPackage* current = package; - /* - string msg = $""" - Total: count={count}, length={Threads.Length}{Environment.NewLine} - """; - /**/ - foreach (MatrixParallelThread thread in pool) - { - long wrapTaskLength = actual; - long ex = extra > 0 ? 1 : 0; - long length = wrapTaskLength + ex; - extra -= ex; - ref long offsetRef = ref Unsafe.As(ref current->PrimTileOffset); - offsetRef += offset; - ref long lengthRef = ref Unsafe.As(ref current->PrimCount); - lengthRef = length; - thread.Tasks.Enqueue((nint)current); - /* - msg += $""" - Slice to thread: total offset={offsetRef} local offset={offset}, length={lengthRef}{Environment.NewLine} - """; - /**/ - current = MatrixParallelFactory.FromExistPackage(current); - offset = length; - } - - // Console.WriteLine(msg); - } - - #region wrap task - - internal unsafe Queue WrapTask { get; set; } = []; - - internal int WrapTaskCapacity { get ; init; } - - #endregion - } - public enum SleepMode : int { @@ -186,4 +45,85 @@ namespace StgSharp.Mathematics.Numeric DeepSleep = ThreadPriority.Lowest, } + + public class MatrixParallelWrap : IDisposable + { + + private MatrixParallelThread[] _threads; + private bool _disposed = false; + private int _count; + private ManualResetEventSlim _leaderEvent = new ManualResetEventSlim(false); + public ThreadPriority AfterWork { get; private set; } + + internal MatrixParallelWrap( + MatrixParallelThread[] threadArray + ) + { + _threads = threadArray; + IsMultithread = threadArray.Length > 1; + _count = threadArray.Length; + } + + public MatrixParallelQueue TaskQueue { get; set; } + + public ReadOnlySpan Threads + { + get + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelWrap).Name); + } + return _threads; + } + } + + private bool IsMultithread { get; init; } + + public void Dispose() + { + if (_disposed) { + return; + } + MatrixParallel.ReturnParallelResources(ref _threads); + _disposed = true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public MatrixParallelThread GetLeader() + { + MatrixParallelThread leader = Threads[1]; + leader.Role = MatrixParallelThread.RoleType.Leader; + leader.LeaderEvent = _leaderEvent; + return leader; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void LaunchParallel( + SleepMode afterWork + ) + { + AfterWork = (ThreadPriority)afterWork; + MatrixParallelThread leader = GetLeader(); + leader.MatrixParallelLeader(); + WaitAllWraps(); + } + + public void WaitAllWraps() + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelWrap).Name); + } + SpinWait.SpinUntil(() => Volatile.Read(ref _count) == 0); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ReportThreadAccomplished() + { + if (_disposed) { + throw new ObjectDisposedException(typeof(MatrixParallelWrap).Name); + } + Interlocked.Decrement(ref _count); + } + + } } diff --git a/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs.old b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs.old new file mode 100644 index 00000000..59d4efa0 --- /dev/null +++ b/StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs.old @@ -0,0 +1,189 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixParallelWrap" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.HighPerformance; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Net.WebSockets; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public unsafe class MatrixParallelWrap + { + + private int _begin; + + private int _remainThreadCount; + private readonly MatrixParallelHandle _handle; + + internal MatrixParallelWrap( + MatrixParallelHandle handle, + int begin, + int count + ) + { + WrapTaskCapacity = count; + _handle = handle; + _begin = begin; + for (int i = 0; i < WrapTaskCapacity; i++) + { + Threads[i].Role = MatrixParallelThread.RoleType.Worker; + Threads[i].BindedWrap = this; + } + Threads[0].Role = MatrixParallelThread.RoleType.Scheduler; + } + + public static SleepMode AfterWork + { + get; + set => Interlocked.Exchange(ref Unsafe.As(ref field), (int)value); + } + + internal int Magic { get; set; } + + private ReadOnlySpan Threads + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => _handle.Threads[_begin..(_begin + WrapTaskCapacity)]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReportThreadAccomplished() + { + Interlocked.Decrement(ref _remainThreadCount); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReportWrapAccomplished() + { + _handle.ReportWrapAccomplished(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetAllWorker() + { + for (int i = 1; i < WrapTaskCapacity; i++) { + Threads[i].Restart(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void WaitAllWorker() + { + SpinWait.SpinUntil(() => Volatile.Read(ref _remainThreadCount) == 0); + } + + internal void PublishTaskToWorkers() + { + Queue tasks = WrapTask; + while (tasks.TryDequeue(out nint handle)) + { + MatrixParallelTaskPackage* task = (MatrixParallelTaskPackage*)handle; + if ((int)task->ComputeMode.OperationStyle > 0) + { + PublishBufferTaskToWorkers(task); + } else + { + throw new NotImplementedException(); + } + } + } + + internal void SetScheduler() + { + MatrixParallelThread thread = Threads[0]; + thread.Restart(); + _ = Interlocked.Exchange(ref _remainThreadCount, WrapTaskCapacity); + } + + private void PublishBufferTaskToWorkers( + MatrixParallelTaskPackage* package + ) + { + ReadOnlySpan pool = Threads; + if (pool.Length == 1) + { + pool[0].Tasks.Enqueue((nint)package); + return; + } + long count = Unsafe.As(ref package->PrimCount); + long actual = count / pool.Length; + long extra = count % pool.Length; + + long offset = 0; + MatrixParallelTaskPackage* current = package; + /* + string msg = $""" + Total: count={count}, length={Threads.Length}{Environment.NewLine} + """; + /**/ + foreach (MatrixParallelThread thread in pool) + { + long wrapTaskLength = actual; + long ex = extra > 0 ? 1 : 0; + long length = wrapTaskLength + ex; + extra -= ex; + ref long offsetRef = ref Unsafe.As(ref current->PrimTileOffset); + offsetRef += offset; + ref long lengthRef = ref Unsafe.As(ref current->PrimCount); + lengthRef = length; + thread.Tasks.Enqueue((nint)current); + /* + msg += $""" + Slice to thread: total offset={offsetRef} local offset={offset}, length={lengthRef}{Environment.NewLine} + """; + /**/ + current = MatrixParallelFactory.FromExistPackage(current); + offset = length; + } + + // Console.WriteLine(msg); + } + + #region wrap task + + internal unsafe Queue WrapTask { get; set; } = []; + + internal int WrapTaskCapacity { get ; init; } + + #endregion + } + + public enum SleepMode : int + { + + QuickWakeup = ThreadPriority.AboveNormal, + NormalSleep = ThreadPriority.BelowNormal, + DeepSleep = ThreadPriority.Lowest, + + } +} diff --git a/StgSharp.Mathematics/Numeric/MatrixTaskFactory.cs b/StgSharp.Mathematics/Numeric/MatrixTaskFactory.cs index 27936f34..0013d367 100644 --- a/StgSharp.Mathematics/Numeric/MatrixTaskFactory.cs +++ b/StgSharp.Mathematics/Numeric/MatrixTaskFactory.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="MatrixTaskFactory" // Project: StgSharp @@ -58,7 +58,7 @@ namespace StgSharp.Mathematics.Numeric private static SlabAllocator _wrapAllocator; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static MatrixParallelTaskPackage* CreateBaseTask() where T : unmanaged, INumber + public static MatrixParallelTaskPackage* CreateBaseTask() { MatrixParallelTaskPackage* p = (MatrixParallelTaskPackage*)_wrapAllocator.Allocate(); Span s = new Span((byte*)p, sizeof(MatrixParallelTaskPackage)); @@ -74,16 +74,9 @@ namespace StgSharp.Mathematics.Numeric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe MatrixParallelTaskPackage* FromExistPackage(MatrixParallelTaskPackage* exist) - where T : unmanaged, INumber - { - MatrixParallelTaskPackage* ptr = (MatrixParallelTaskPackage*)_wrapAllocator.Allocate(); - MatrixParallelTaskPackage.Copy(exist, ptr); - return ptr; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe MatrixParallelTaskPackage* FromExistPackage(MatrixParallelTaskPackage* exist) + public static unsafe MatrixParallelTaskPackage* FromExistPackage( + MatrixParallelTaskPackage* exist + ) { MatrixParallelTaskPackage* ptr = (MatrixParallelTaskPackage*)_wrapAllocator.Allocate(); MatrixParallelTaskPackage.Copy(exist, ptr); @@ -92,22 +85,17 @@ namespace StgSharp.Mathematics.Numeric [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Release(ScalarPacket* source) + public static void Release( + ScalarPacket* source + ) { _scalarAllocator.Free((nuint)source); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Release(MatrixParallelTaskPackage* source) - { - if (source->Scalar != null) { - _scalarAllocator.Free((nuint)source->Scalar); - } - _wrapAllocator.Free((nuint)source); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Release(MatrixParallelTaskPackage* source) where T : unmanaged, INumber + public static void Release( + MatrixParallelTaskPackage* source + ) { if (source->Scalar != null) { _scalarAllocator.Free((nuint)source->Scalar); diff --git a/StgSharp.Native/SIMDID.x86.md b/StgSharp.Native/SIMDID.x86.md index 892902ca..08050769 100644 --- a/StgSharp.Native/SIMDID.x86.md +++ b/StgSharp.Native/SIMDID.x86.md @@ -1,8 +1,8 @@ # SIMDID (x86-64) 64-bit Specification (Completed per fixed layout) -This document completes the **64-bit SIMDID** bit layout as specified, with emphasis on the design of the **AVX512 Feature section ([31..16])**, so it can cover the messy AVX-512 subsets while keeping branching logic compact. It also allocates an **AVX Feature byte** adjacent to the AVX512/AMX regions for AVX-family capabilities. +This document completes the **64-bit SIMDID** bit layout as specified, with emphasis on the design of the **AVX512 Feature section ([31..16])**, so it can cover the messy AVX-512 subsets while keeping branching logic compact. It also allocates an **AVX Feature byte** adjacent to the AVX512/AMX regions for AVX-family capabilities, and an **AVX10 Feature byte** for the converged AVX10 ISA. -> Convention: bit0 is the least significant bit (LSB). Only x86-64 interpreter semantics are defined here; the header’s Root ISA allows extension to other architectures. +> Convention: bit0 is the least significant bit (LSB). Only x86-64 interpreter semantics are defined here; the header's Root ISA allows extension to other architectures. --- @@ -10,10 +10,10 @@ This document completes the **64-bit SIMDID** bit layout as specified, with emph ``` SIMDID (64 bits) [63 ........................................................ 0] -+----------+-----------+----------------+----------------+----------------+----------------+----------------+ -| [63..56] | [47..40] | [39..32] | [31..16] | [15..8] | [7..4] | [3..0] | -| uArchHi | AVX Feat | AMX Future | AVX512 Feature | MainLevel | Manuf | Root | -+----------+-----------+----------------+----------------+----------------+----------------+----------------+ ++----------+-----------+-----------+----------------+----------------+----------------+--------+-------+ +| [63..56] | [55..48] | [47..40] | [39..32] | [31..16] | [15..8] | [7..4] | [3..0]| +| uArchHi | AVX10Feat | AVX Feat | AMX Future | AVX512 Feature | MainLevel | Manuf | Root | ++----------+-----------+-----------+----------------+----------------+----------------+--------+-------+ ``` - **[3..0] Root ISA**: Root ISA type (selects interpreter) @@ -22,6 +22,7 @@ SIMDID (64 bits) [63 ........................................................ 0 - **[31..16] AVX512 Feature**: AVX-512 features/subsets/implementation form (key part) - **[39..32] AMX Future**: AMX features (few bits for now) - **[47..40] AVX Feature**: AVX-family capabilities (base requires AVX+AVX2; key extensions include FMA/F16C) +- **[55..48] AVX10 Feature**: AVX10 converged ISA version and vector width support - **[63..56] uArchHi**: Optional microarchitecture/policy hints (default 0, no functional impact). --- @@ -36,12 +37,11 @@ Suggested encoding (minimal usable set, expandable if needed): | 1 | x86-64 | | 2 | AArch64 | | 3 | PPC64LE | -| 4 | Apple Silicon (distinct value as required) | -| 5 | RISC-V | -| 6 | LoongArch | -| 7..15 | Reserved | +| 4 | RISC-V | +| 5 | LoongArch | +| 6..15 | Reserved | -> Note: Architecturally Apple Silicon is AArch64; a distinct value is provided here because the Root ISA layer may distinguish it. Interpreter choice is up to implementation. +> Note: Apple Silicon is architecturally AArch64 and uses `Root ISA = 2`. Apple vs Qualcomm vs other AArch64 vendors are distinguished via the **Manufacture** nibble under the AArch64 interpreter, not via a separate Root ISA value. --- @@ -57,7 +57,18 @@ For **x86-64**: | 3 | VIA/Zhaoxin | | 4..15 | Reserved | -For other Root ISAs, this nibble is defined by their interpreters (e.g., Apple/Qualcomm/MediaTek for ARM). +For **AArch64**: + +| Value | Manufacture | +|---:|-------------| +| 0 | Unknown/Other | +| 1 | Apple | +| 2 | Qualcomm | +| 3 | MediaTek | +| 4 | Ampere | +| 5..15 | Reserved | + +For other Root ISAs, this nibble is defined by their interpreters. --- @@ -74,10 +85,10 @@ For other Root ISAs, this nibble is defined by their interpreters (e.g., Apple/Q | 4 | AVX2_FMA_F16C (from this level onward, 16-bit types are supported) | | 5 | AVX512 (check [31..16]) | | 6 | AMX | -| 7 | AVX10 | +| 7 | AVX10 (check [55..48]) | | 8..255 | Reserved | -> Scheduling suggestion: `MainLevel` is the primary switch; only when `MainLevel >= AVX512` should `AVX512 Feature` be parsed; only when `MainLevel >= AMX` should `AMX Future` be parsed. AVX Feature sits beside AMX/AVX512 and is meaningful when AVX or higher is present. +> Scheduling suggestion: `MainLevel` is the primary switch; only when `MainLevel >= AVX512` should `AVX512 Feature` be parsed; only when `MainLevel >= AMX` should `AMX Future` be parsed; only when `MainLevel >= AVX10` should `AVX10 Feature` be parsed. AVX Feature sits beside AMX/AVX512 and is meaningful when AVX or higher is present. --- @@ -109,7 +120,7 @@ Notes: | 2 | AVX512-NATIVE (full 512-bit ALU; 512-bit path is usable) | | 3 | Reserved | -- **VBMI/VBMI2/VNNI/BF16/FP16 ([29..26])**: Commonly needed niche or AI-related small-type extensions; set as needed. +- **VBMI/VBMI2/VNNI/BF16/FP16 ([29..25])**: Commonly needed niche or AI-related small-type extensions; set as needed. - **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions. - **[31..30], [21], [19..16]**: Reserved for future allocation. @@ -134,7 +145,39 @@ Rules: --- -## 8) "AVX512BASE" Determination (Single Check) +## 8) AVX10 Feature Byte ([55..48], 8 bits) + +Purpose: encode the converged AVX10 ISA version and supported vector widths. AVX10 unifies and replaces the fragmented AVX-512 subset model; a single version number implies a defined set of instructions (e.g., AVX10.1 implies F+CD+VL+DQ+BW+VBMI+VBMI2+VNNI+BF16+FP16 converged). The width bits indicate what maximum vector width the hardware actually implements. + +**CPUID detection**: `CPUID leaf 0x24, subleaf 0`. +- EBX[7:0] = AVX10 converged ISA version number (≥ 1 if supported). +- EBX[16] = 128-bit vector support (always 1 when AVX10 is present). +- EBX[17] = 256-bit vector support. +- EBX[18] = 512-bit vector support. + +AVX10 Feature (bits 55..48) + +| bit | 55 | 54 | 53 | 52 | 51 | 50 | 49 | 48 | +|-----|----|----|----| ---- | ---- |----|----|------| +| use | R | R | R | 512W | 256W | V2 | V1 | AVX10 | + +Rules: +- **AVX10 (bit 48)**: CPU declares AVX10 converged ISA support (leaf 0x24 reports a valid version ≥ 1). +- **V1 (bit 49)**: AVX10 version ≥ 1 (AVX10.1). Set when `EBX[7:0] >= 1`. +- **V2 (bit 50)**: AVX10 version ≥ 2 (AVX10.2). Set when `EBX[7:0] >= 2`. +- **256W (bit 51)**: Hardware supports 256-bit vector width for AVX10 instructions (`EBX[17]` == 1). +- **512W (bit 52)**: Hardware supports 512-bit vector width for AVX10 instructions (`EBX[18]` == 1). +- **[53..55]**: Reserved for future AVX10 versions (V3, V4...) or additional width/feature bits. + +> Note: When `AVX10 == 1`, the 128-bit width is always implied and does not need a dedicated bit. `V1` and `V2` are cumulative — if `V2` is set, `V1` must also be set. + +**Relationship with AVX512 Feature**: +- AVX10 is a superset specification. When `MainLevel == AVX10`, the `AVX512 Feature` field may still be populated for backward-compatible code paths (e.g., checking `AVX512_IMPL` to select DUAL vs NATIVE 512-bit policy). +- AVX10/256 hardware does **not** support 512-bit operations; code should check `512W` before using zmm registers. + +--- + +## 9) "AVX512BASE" Determination (Single Check) - **Rule**: `BASE=1` implies `F+CD+VL+DQ+BW` are all present; no need for per-subset checks. Detection must keep this consistent; if not all subsets are present, set to 0. - **Runtime branching**: @@ -143,10 +186,11 @@ Rules: - Check `BASE`; if 0, fall back to AVX2/FMA (or treat as detection failure). - Check `AVX512_IMPL`: `DUAL` uses 256-bit/downclock-safe path, `NATIVE` uses 512-bit path. - When small-type/AI capabilities are required, consult `VNNI/BF16/FP16/VBMI/VBMI2` bits. + - If `MainLevel == AVX10`: prefer AVX10 Feature for capability checks; `AVX512 Feature` remains valid for legacy branching. --- -## 9) AMX Future ([39..32], 8 bits) +## 10) AMX Future ([39..32], 8 bits) A single byte is reserved; define three common bits for now, leave the rest reserved: @@ -161,7 +205,7 @@ When `MainLevel < AMX`, this byte can be all zeros. --- -## 10) uArchHi ([63..56], 8 bits) +## 11) uArchHi ([63..56], 8 bits) Purpose: optional high-byte strategy field to address issues like early Intel/AMD "glue" AVX512 that are not expressible purely via CPUID subsets. Defaults to 0. @@ -177,7 +221,7 @@ Practical strategy for a single maintainer: --- -## 11) Minimal Detection-Side Fill Rules (x86-64) +## 12) Minimal Detection-Side Fill Rules (x86-64) 1) Set `RootISA = x86-64`, `Manufacture = Intel/AMD`. 2) Compute `MainLevel` (per ascending rule). @@ -189,10 +233,17 @@ Practical strategy for a single maintainer: - When VNNI/BF16/FP16/VBMI/VBMI2 are detected, set the corresponding bits. 4) If `MainLevel >= AMX`: fill the AMX byte. 5) When AVX/AVX2 are present, populate the AVX Feature byte: set AVX and AVX2; set FMA and F16C when detected; leave reserved bits for future extensions. -6) Use uArchHi only when additional policy is needed; otherwise leave as 0. +6) **AVX10 detection** (requires `CPUID leaf 7 subleaf 1, EDX[19] == 1` as the AVX10 convergence bit): + - If AVX10 is indicated, query `CPUID leaf 0x24, subleaf 0`: + - Read `EBX[7:0]` for version number; set `AVX10`, `V1`, and optionally `V2`. + - Read `EBX[17]` for 256-bit support → set `256W`. + - Read `EBX[18]` for 512-bit support → set `512W`. + - Promote `MainLevel` to `AVX10` (value 7). + - The `AVX512 Feature` field should still be filled for backward compatibility. +7) Use uArchHi only when additional policy is needed; otherwise leave as 0. --- -## 12) Choice for High Bits Reservation +## 13) Choice for High Bits Reservation -Currently `[31..24]` is reserved for AVX512 extensions/AI. `[47..44]` is reserved for AVX-family extensions beside the AVX byte. uArchHi `[63..56]` remains available for minimal policy hints. If IFMA or other extensions are more important, repurpose the reserved bits while keeping `BASE` and `Impl` unchanged. \ No newline at end of file +`[31..24]` is reserved for AVX512 extensions/AI. `[47..44]` is reserved for AVX-family extensions beside the AVX byte. `[55..53]` is reserved for future AVX10 versions. uArchHi `[63..56]` remains available for minimal policy hints. If IFMA or other extensions are more important, repurpose the reserved bits while keeping `BASE`, `Impl`, and `AVX10` core bits unchanged. \ No newline at end of file diff --git a/StgSharp.Native/StgSharp.Native.vcxproj b/StgSharp.Native/StgSharp.Native.vcxproj index 5c24176a..0944cd4d 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj +++ b/StgSharp.Native/StgSharp.Native.vcxproj @@ -28,21 +28,21 @@ true Unicode false - v145 + v143 DynamicLibrary false true Unicode - v145 + v143 DynamicLibrary false true Unicode - v145 + v143 @@ -171,6 +171,7 @@ + @@ -194,6 +195,7 @@ + diff --git a/StgSharp.Native/StgSharp.Native.vcxproj.filters b/StgSharp.Native/StgSharp.Native.vcxproj.filters index c6ac261a..bb28e3c8 100644 --- a/StgSharp.Native/StgSharp.Native.vcxproj.filters +++ b/StgSharp.Native/StgSharp.Native.vcxproj.filters @@ -66,6 +66,9 @@ 源文件 + + 源文件 + @@ -120,5 +123,8 @@ Document + + Document + \ No newline at end of file diff --git a/StgSharp.Native/sn-head/StgSharpNative.h b/StgSharp.Native/sn-head/StgSharpNative.h index 3e731be3..1c071ce6 100644 --- a/StgSharp.Native/sn-head/StgSharpNative.h +++ b/StgSharp.Native/sn-head/StgSharpNative.h @@ -65,7 +65,12 @@ SN_API GLFWglproc SN_DECL loadGlfuncDefault(char *procName); SN_API void SN_DECL unloadImageData(Image *out); SN_API char *SN_DECL readLog(void); SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table); -SN_API SIMDID SN_DECL load_intrinsic_function(void *intrinsic_context); +SN_API SIMDID SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id); +SN_API SIMDID SN_DECL sn_get_simd_level(); +/* Per-core variant: always re-executes CPUID, never caches. + * Must be called on the target core (after affinity is set). */ +SN_API SIMDID SN_DECL sn_get_simd_level_local(); +SN_API void SN_DECL sn_set_thread_affinity(int core_id); #ifdef __cplusplus } diff --git a/StgSharp.Native/sn-head/sn_intrincontext.x86.h b/StgSharp.Native/sn-head/sn_intrincontext.x86.h index 99672e87..2801609d 100644 --- a/StgSharp.Native/sn-head/sn_intrincontext.x86.h +++ b/StgSharp.Native/sn-head/sn_intrincontext.x86.h @@ -12,6 +12,7 @@ // bits [31..16]: AVX512 Feature // bits [39..32]: AMX Feature // bits [47..40]: AVX Feature +// bits [55..48]: AVX10 Feature // bits [63..56]: uArchHi / Policy (optional) // ----------------------------------------------------------------------------- #define SIMDID_SHIFT_ROOT 0 @@ -20,6 +21,7 @@ #define SIMDID_SHIFT_AVX512 16 #define SIMDID_SHIFT_AMX 32 #define SIMDID_SHIFT_AVX 40 +#define SIMDID_SHIFT_AVX10 48 #define SIMDID_SHIFT_UARCH 56 #define SIMDID_MASK_ROOT 0xFULL @@ -28,6 +30,7 @@ #define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512) #define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX) #define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX) +#define SIMDID_MASK_AVX10 (0xFFULL << SIMDID_SHIFT_AVX10) #define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH) // Root ISA (low nibble) @@ -35,9 +38,8 @@ #define SIMDID_ROOT_X86_64 0x1 #define SIMDID_ROOT_AARCH64 0x2 #define SIMDID_ROOT_PPC64LE 0x3 -#define SIMDID_ROOT_APPLESILICON 0x4 -#define SIMDID_ROOT_RISCV64 0x5 -#define SIMDID_ROOT_LOONGARCH 0x6 +#define SIMDID_ROOT_RISCV64 0x4 +#define SIMDID_ROOT_LOONGARCH 0x5 // Manufacture (x86-64 mapping; other ISAs may reinterpret) #define SIMDID_MANU_UNKNOWN 0x0 @@ -54,31 +56,39 @@ #define SIMDID_MAIN_LVL_AVX10 0x05u #ifndef SIMDID_AVX512_BASE -#define SIMDID_AVX512_FP16 (1U << 29) -#define SIMDID_AVX512_BF16 (1U << 28) -#define SIMDID_AVX512_VNNI (1U << 27) -#define SIMDID_AVX512_VBMI2 (1U << 26) -#define SIMDID_AVX512_VBMI (1U << 25) -#define SIMDID_AVX512_AI_RESERVED (1U << 24) -#define SIMDID_AVX512_BASE (1U << 20) -#define SIMDID_AVX512_IMPL_SHIFT 22 -#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT) -#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_FP16 (1U << 13) /* relative bit 13 within [31..16] */ +#define SIMDID_AVX512_BF16 (1U << 12) +#define SIMDID_AVX512_VNNI (1U << 11) +#define SIMDID_AVX512_VBMI2 (1U << 10) +#define SIMDID_AVX512_VBMI (1U << 9) +#define SIMDID_AVX512_AI_RESERVED (1U << 8) +#define SIMDID_AVX512_BASE (1U << 4) +#define SIMDID_AVX512_IMPL_SHIFT 6 +#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT) +#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT) #endif #ifndef SIMDID_AVX_BASE -#define SIMDID_AVX (1ULL << 40) -#define SIMDID_AVX2 (1ULL << 41) -#define SIMDID_AVX_FMA (1ULL << 42) -#define SIMDID_AVX_F16C (1ULL << 43) +#define SIMDID_AVX (1U << 0) /* relative bit 0 within [47..40] */ +#define SIMDID_AVX2 (1U << 1) +#define SIMDID_AVX_FMA (1U << 2) +#define SIMDID_AVX_F16C (1U << 3) #endif #ifndef SIMDID_AMX_TILE -#define SIMDID_AMX_TILE (1ULL << 32) -#define SIMDID_AMX_INT8 (1ULL << 33) -#define SIMDID_AMX_BF16 (1ULL << 34) +#define SIMDID_AMX_TILE (1U << 0) /* relative bit 0 within [39..32] */ +#define SIMDID_AMX_INT8 (1U << 1) +#define SIMDID_AMX_BF16 (1U << 2) +#endif + +#ifndef SIMDID_AVX10_BASE +#define SIMDID_AVX10 (1U << 0) /* relative bit 0 within [55..48] */ +#define SIMDID_AVX10_V1 (1U << 1) +#define SIMDID_AVX10_V2 (1U << 2) +#define SIMDID_AVX10_256W (1U << 3) +#define SIMDID_AVX10_512W (1U << 4) #endif // Pack helpers (define locally to avoid missing macros when included standalone) @@ -90,6 +100,7 @@ (((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512) #define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX) #define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX) +#define SIMDID_PACK_AVX10(avx10_bits) (((uint64_t)(avx10_bits) & 0xFFULL) << SIMDID_SHIFT_AVX10) #define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH) #endif diff --git a/StgSharp.Native/sn-head/sn_target.h b/StgSharp.Native/sn-head/sn_target.h index d818f596..86e7a301 100644 --- a/StgSharp.Native/sn-head/sn_target.h +++ b/StgSharp.Native/sn-head/sn_target.h @@ -59,6 +59,4 @@ typedef union SIMDID { char make_byte[8]; } SIMDID; -SN_API SIMDID SN_DECL sn_get_simd_level(); - #endif // SN_TARGET diff --git a/StgSharp.Native/sn-src/cpuid.x86.c b/StgSharp.Native/sn-src/cpuid.x86.c index 9b499b71..356ec70c 100644 --- a/StgSharp.Native/sn-src/cpuid.x86.c +++ b/StgSharp.Native/sn-src/cpuid.x86.c @@ -1,5 +1,6 @@ #include "sn_target.h" #include "sn_intrinsic.h" +#include "StgSharpNative.h" static SIMDID global_simdid = { 0 }; @@ -76,11 +77,9 @@ static int is_intel_native_avx512(int family, int model) } } -SN_API SIMDID SN_DECL sn_get_simd_level() +SN_API SIMDID SN_DECL sn_get_simd_level_local() { - if (global_simdid.mask) { - return global_simdid; - } + SIMDID _simdid = { 0 }; int eax = 0, ebx = 0, ecx = 0, edx = 0; @@ -114,6 +113,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level() uint64_t avx512_bits = 0; uint64_t amx_bits = 0; uint64_t avx_bits = 0; + uint64_t avx10_bits = 0; /* Baseline main level */ if (has_sse42) @@ -135,15 +135,17 @@ SN_API SIMDID SN_DECL sn_get_simd_level() const int has_vbmi = (ecx & (1 << 1)) != 0; const int has_vbmi2 = (ecx & (1 << 6)) != 0; - /* leaf 7, subleaf 1 for BF16/FP16 */ + /* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 — capture before + overwriting regs with subleaf 1 */ + const int has_amx_tile = (edx & (1 << 24)) != 0; + const int has_amx_int8 = (edx & (1 << 25)) != 0; + const int has_amx_bf16 = (edx & (1 << 22)) != 0; + + /* leaf 7, subleaf 1 for BF16/FP16/AVX10 convergence */ get_cpuid(7, 1, &eax, &ebx, &ecx, &edx); const int has_bf16 = (eax & (1 << 5)) != 0; /* AVX512_BF16 */ const int has_fp16 = (eax & (1 << 23)) != 0; /* AVX512_FP16 */ - - /* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 */ - const int has_amx_tile = (edx & (1 << 24)) != 0; - const int has_amx_int8 = (edx & (1 << 25)) != 0; - const int has_amx_bf16 = (edx & (1 << 26)) != 0; + const int has_avx10 = (edx & (1 << 19)) != 0; /* AVX10 convergence */ if (has_avx2) { avx_bits |= SIMDID_AVX2; @@ -194,11 +196,39 @@ SN_API SIMDID SN_DECL sn_get_simd_level() amx_bits |= SIMDID_AMX_BF16; main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX); } + + /* AVX10 detection — requires AVX10 convergence bit + (leaf 7 subleaf 1 EDX[19]) and leaf 0x24 */ + if (has_avx10) { + int a10_eax = 0, a10_ebx = 0, a10_ecx = 0, a10_edx = 0; + get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx); + const int avx10_ver = a10_ebx & 0xFF; + if (avx10_ver >= 1) { + avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1; + if (avx10_ver >= 2) + avx10_bits |= SIMDID_AVX10_V2; + if (a10_ebx & (1 << 17)) + avx10_bits |= SIMDID_AVX10_256W; + if (a10_ebx & (1 << 18)) + avx10_bits |= SIMDID_AVX10_512W; + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10); + } + } } } - global_simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) | - SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits); + _simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) | + SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits) | + SIMDID_PACK_AVX10(avx10_bits); + return _simdid; +} + +SN_API SIMDID SN_DECL sn_get_simd_level() +{ + if (global_simdid.mask) { + return global_simdid; + } + global_simdid = sn_get_simd_level_local(); return global_simdid; } diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index 9a1698f1..b0ecf588 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -14,14 +14,21 @@ #endif // _MSC_VER PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id); +PRIVATE void load_intrinsic_avx_dual(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id); +PRIVATE void load_intrinsic_512_dual(sn_intrinsic *context, SIMDID id); PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id); -SN_API SIMDID load_intrinsic_function(void *intrinsic_context) +SN_API SIMDID load_intrinsic_function(void *intrinsic_context, uint64_t id) { sn_null_assert(intrinsic_context); - SIMDID simdid = sn_get_simd_level(); + SIMDID simdid; + if (id) { + simdid.mask = id; + } else { + simdid = sn_get_simd_level(); + } sn_intrinsic *context = (sn_intrinsic *)intrinsic_context; @@ -33,6 +40,8 @@ SN_API SIMDID load_intrinsic_function(void *intrinsic_context) load_intrinsic_avx(context, simdid); break; case SIMDID_MAIN_LVL_AVX512: + case SIMDID_MAIN_LVL_AMX: + case SIMDID_MAIN_LVL_AVX10: load_intrinsic_512(context, simdid); break; } @@ -75,8 +84,8 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) { char avx_seg = id.make_byte[5]; - char is_fma = (avx_seg & 1 << 3) != 0; - char is_f16c = (avx_seg & 1 << 4) != 0; + char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0; + char is_f16c = (avx_seg & SIMDID_AVX_F16C) != 0; context->city_hash_simplify = city_hash_simplify_sse; context->factorial_simd = factorial_simd_sse; diff --git a/StgSharp.Native/sn-src/m.fma_tile.x86.c b/StgSharp.Native/sn-src/m.fma_tile.x86.c index 665ffdac..9db7d0de 100644 --- a/StgSharp.Native/sn-src/m.fma_tile.x86.c +++ b/StgSharp.Native/sn-src/m.fma_tile.x86.c @@ -7,34 +7,8 @@ #define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i) -/* -[MethodImpl(MethodImplOptions.AggressiveInlining)] -internal static unsafe MatrixKernel* GetKernelAddressUnsafe( - MatrixKernel* buffer, - int colLength, - int x, - int y -) where T : unmanaged, INumber -{ - return buffer + (colLength * x + y); -} -*/ - #define GET_INDEX(col_len, x, y) (col_len * x + y) -/* -for (int k = 0; k < commonK; k += 1) { - NativeIntrinsic.Context - .mat[f64].kernel_fma( - (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)leftBuffer, - ansHeight, k, j), - (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)rightBuffer, - commonK, j, k), - (MatrixPanel*)GetKernelAddressUnsafe((MatrixKernel*)ansBuffer, - ansHeight, i, j)); -} -*/ - // float DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma) diff --git a/StgSharp.Native/sn-src/t.affinity.c b/StgSharp.Native/sn-src/t.affinity.c new file mode 100644 index 00000000..6d6d0d7c --- /dev/null +++ b/StgSharp.Native/sn-src/t.affinity.c @@ -0,0 +1,31 @@ +#include "sn_target.h" +#include "StgSharpNative.h" + +#if SN_IS_OS(SN_OS_LINUX) +#define _GNU_SOURCE +#include +#include +#endif + +/* Pin the calling thread to a single logical processor. + * core_id: zero-based logical processor index. + * Silently ignored if core_id is out of range or the OS call fails. */ +SN_API void SN_DECL sn_set_thread_affinity(int core_id) +{ + if (core_id < 0) + return; + +#if SN_IS_OS(SN_OS_WINDOWS) + /* windows.h already included via sn_internal.h */ + if (core_id >= 64) + return; + DWORD_PTR mask = (DWORD_PTR)1 << core_id; + SetThreadAffinityMask(GetCurrentThread(), mask); + +#elif SN_IS_OS(SN_OS_LINUX) + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(core_id, &set); + pthread_setaffinity_np(pthread_self(), sizeof(set), &set); +#endif +} \ No newline at end of file