Arrange code structure. Introduce dynamic scheduler and multi leader.

StgSharp.Common/Collections/BidirectionalDictionary.cs

@@ -32,7 +32,7 @@ using System.Collections.ObjectModel;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
-namespace StgSharp.Commom.Collections
+namespace StgSharp.Collections
 {
     public sealed class BidirectionalDictionary<TFirst, TSecond> : IBidirectionalDictionary<TFirst, TSecond>
     {

StgSharp.Common/Collections/CapacityFixedBag.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="CachePool"
+// file="CapacityFixedBag"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -25,36 +25,57 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-using StgSharp.Graphics.OpenGL;
-
 using System;
 using System.Collections;
-using System.Collections.Concurrent;
 using System.Collections.Generic;
-using System.Text;
 
-namespace StgSharp.Commom.Collections
+namespace StgSharp.Collections
 {
-    /*
+    public sealed class CapacityFixedBag<T> : IEnumerable<T>
-    public interface IPoolItem<TSignature>
     {
 
-        public int ID
+        private readonly T[] _buffer;
+
+        public CapacityFixedBag(int capacity)
         {
-            get;
+            _buffer = new T[capacity];
-            init;
+        }
+
+        public int Count { get; set; }
+
+        public int Capacity => _buffer.Length;
+
+        public void Add(T item)
+        {
+            if (Count >= _buffer.Length) {
+                throw new InvalidOperationException("Bag capacity exceeded.");
+            }
+            _buffer[Count] = item;
+            Count++;
+        }
+
+        public IEnumerator<T> GetEnumerator()
+        {
+            for (int i = 0; i < Count; i++) {
+                yield return _buffer[i];
+            }
+        }
+
+        public void Remove(T item)
+        {
+            int index = Array.IndexOf(_buffer, item, 0, Count);
+            if (index >= 0)
+            {
+                _buffer[index] = _buffer[Count - 1];
+                _buffer[Count - 1] = default!;
+                Count--;
+            }
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
         }
 
     }
-
-    public class CacheIndexPool<TSignature, TItem> : IEnumerable<TItem>
-        where TItem: IPoolItem<TSignature>
-    {
-
-        public CacheIndexPool(
-                       Func<TSignature> 
- ) { }
-
-    }
-    /**/
 }

StgSharp.Common/Collections/CapacityFixedList.cs

@@ -28,11 +28,9 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Reflection;
 using System.Runtime.CompilerServices;
-using System.Text;
 
-namespace StgSharp.Commom.Collections
+namespace StgSharp.Collections
 {
     public static class CapacityFixedListBuilder
     {

StgSharp.Common/Collections/CapacityFixedStack.cs

@@ -31,7 +31,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
 
-namespace StgSharp.Commom.Collections
+namespace StgSharp.Collections
 {
     [CollectionBuilder(typeof(CapacityFixedStackBuilder), nameof(CapacityFixedStackBuilder.Create))]
     public class CapacityFixedStack<T> : IEnumerable<T>
@@ -72,13 +72,7 @@ namespace StgSharp.Commom.Collections
 
         public T Peek()
         {
-            if (_index >= 0)
+            return _index >= 0 ? _values[_index] : throw new InvalidOperationException("Stack is empty.");
-            {
-                return _values[_index];
-            } else
-            {
-                throw new InvalidOperationException("Stack is empty.");
-            }
         }
 
         public T Pop()
@@ -86,6 +80,7 @@ namespace StgSharp.Commom.Collections
             if (_index >= 0)
             {
                 T item = _values[_index];
+                _index--;
                 return item;
             } else
             {
@@ -95,13 +90,9 @@ namespace StgSharp.Commom.Collections
 
         public void Push(T item)
         {
-            if (_index < _values.Length - 1)
+            _values[++_index] = _index < _values.Length - 1 ?
-            {
+                                item :
-                _values[++_index] = item;
+                                throw new InvalidOperationException("Stack is full.");
-            } else
-            {
-                throw new InvalidOperationException("Stack is full.");
-            }
         }
 
         public void PushRange(ReadOnlySpan<T> span)
@@ -115,6 +106,21 @@ namespace StgSharp.Commom.Collections
             _index += span.Length;
         }
 
+        public  bool TryPop(out T value)
+        {
+            if (_index >= 0)
+            {
+                T item = _values[_index];
+                _index--;
+                value = item;
+                return true;
+            } else
+            {
+                value = default!;
+                return false;
+            }
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal Span<T> GetValueLayout()
         {

StgSharp.Common/Collections/IBidrirectionalDictionary.cs

@@ -30,7 +30,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 
-namespace StgSharp.Commom.Collections
+namespace StgSharp.Collections
 {
     public interface IBidirectionalDictionary<TFirst, TSecond> : IDictionary<TFirst, TSecond>
     {

StgSharp.Common/Graphics/OpenGL/Buffer/AutoTexture.cs

@@ -25,7 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-using StgSharp.Commom.Collections;
+using StgSharp.Collections;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;

StgSharp.Mathematics/Numeric/MatrixComputeModel.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="MatrixParallel.Execute"
+// file="MatrixComputeModel"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -30,19 +30,24 @@ using System.Runtime.CompilerServices;
 
 namespace StgSharp.Mathematics.Numeric
 {
-    public static partial class MatrixParallel
+    internal static class MatrixComputeModel
     {
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static unsafe void Compute_Binary(MatrixParallelTaskPackageNonGeneric* p)
         {
             int eSize = p->ElementSize;
+
+            // base pointer to matrix kernels
             nint l = p->Left, r = p->Right, a = p->Result;
             long
+
+                // set beginning value to counters for primary offsets
                 lCount = (long)p->LeftPrimOffset * p->LeftPrimStride,
                 rCount = (long)p->RightPrimOffset * p->RightPrimStride,
                 aCount = (long)p->ResultPrimOffset * p->ResultPrimStride,
                 pCount = p->PrimCount,
+                //edge to reset counters
                 lLimit = pCount - p->LeftPrimOffset,
                 rLimit = pCount - p->RightPrimOffset,
                 aLimit = pCount - p->ResultPrimOffset;
@@ -50,9 +55,13 @@ namespace StgSharp.Mathematics.Numeric
                     (delegate* unmanaged[Cdecl]<IntPtr, IntPtr, IntPtr, void>)p->ComputeHandle;
             for (long i = 0; i < pCount; i++)
             {
+                // update primary counters
+                // if reachs edge, reset to zero
                 lCount = i < lLimit ? lCount + p->LeftPrimOffset : 0;
                 rCount = i < rLimit ? rCount + p->RightPrimOffset : 0;
                 aCount = i < rLimit ? aCount + p->ResultPrimOffset : 0;
+
+                // get kernel column pointers
                 nint left = l + (nint)(lCount * eSize);
                 nint right = r + (nint)(rCount * eSize);
                 nint ans = a + (nint)(aCount * eSize);

StgSharp.Mathematics/Numeric/MatrixParallel.Publish.cs

@@ -25,7 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-using StgSharp.Commom.Collections;
+using StgSharp.Collections;
 using StgSharp.HighPerformance;
 
 using System;
@@ -837,7 +837,9 @@ namespace StgSharp.Mathematics.Numeric
                 bool isLargeSlice = extraTiles > 0;
 
                 // count tiles for wrap
-                int toSlice = wrap.WrapTaskCapacity * slicingPerWrap;
+                // TODO 此处应该是线程数量而不是线程束数量
+                // TODO 需要想办法正确同步线程数量信息
+                int toSlice = hasSlice ? wrap.WrapTaskCapacity * slicingPerWrap : 0;
                 int baseTile = wrap.WrapTaskCapacity * tilePerThreadLess;
                 int tileInWrap = baseTile + toSlice + (isLargeSlice ? 1 : 0);
                 int columnInWrap = tileInWrap * tileWidth;

StgSharp.Mathematics/Numeric/MatrixParallel.cs

@@ -25,7 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-using StgSharp.Commom.Collections;
+using StgSharp.Collections;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
@@ -189,175 +189,6 @@ namespace StgSharp.Mathematics.Numeric
             }
         }
 
-        public unsafe class MatrixParallelWrap
-        {
-
-            private readonly Queue<IntPtr>[] _cache;
-            private readonly Thread[] _threads;
-            private readonly ManualResetEventSlim[] _workerReset;
-
-            public MatrixParallelWrap(int id, int wrapSize, bool isFirst)
-            {
-                ID = id;
-                int begin = isFirst ? 1 : 0, end = wrapSize - 1;
-                _cache = new Queue<IntPtr>[wrapSize];
-                _workerReset = new ManualResetEventSlim[end];
-                _threads = new Thread[wrapSize];
-                for (int i = 0; i < end; i++) {
-                    _workerReset[i] = new ManualResetEventSlim(false);
-                }
-                for (int i = 0; i < _cache.Length; i++) {
-                    _cache[i] = new Queue<IntPtr>(8);
-                }
-                for (int i = begin; i < end; i++)
-                {
-                    _threads[i] = new Thread(MatrixParallelWorker)
-                    {
-                        IsBackground = true,
-                        Priority = ThreadPriority.BelowNormal,
-                        Name = $"MPT#{ID}_{i}"
-                    };
-                    _threads[i].Start(new Handle(i, this));
-                }
-                _threads[end] = new Thread(MatrixParallelScheduler)
-                {
-                    IsBackground = true,
-                    Priority = ThreadPriority.BelowNormal,
-                    Name = $"MPT#{ID}_{end}*"
-                };
-                _threads[end].Start(new Handle(end, this));
-            }
-
-            public int ID { get; init; }
-
-            internal ManualResetEventSlim SchedulerReset { get; } = new(false);
-
-            public Span<IntPtr> EnqueuePackage(Span<IntPtr> packageSpan)
-            {
-                int idx = 0;
-                for (; idx < _cache.Length; idx++) {
-                    _cache[idx].Enqueue(packageSpan[idx]);
-                }
-                return packageSpan[idx..];
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public ManualResetEventSlim GetWorkerResetEvent(int id)
-            {
-                return _workerReset[id];
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public Queue<IntPtr> GetWorkerTaskQueue(int idInWrap)
-            {
-                return _cache[idInWrap];
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void SetAllWorker()
-            {
-                foreach (ManualResetEventSlim item in _workerReset) {
-                    item.Set();
-                }
-            }
-
-            public record class Handle(int InWrapIndex, MatrixParallelWrap Wrap);
-
-            #region wrap task
-
-            internal MatrixParallelTaskPackageNonGeneric* WrapTask { get; set; }
-
-            internal int WrapTaskSliceCount { get; set; }
-
-            internal int WrapTaskCapacity
-            {
-                [MethodImpl(MethodImplOptions.AggressiveInlining)]
-                get => _threads.Length;
-            }
-
-                #endregion
-
-
-            #region ThreadOperation
-
-            private static unsafe void MatrixParallelWorker([AllowNull] object obj)
-            {
-                Handle handle = (obj as Handle)!;
-                MatrixParallelWrap wrap = handle.Wrap;
-                int inWrapIndex = handle.InWrapIndex;
-                Thread self = Thread.CurrentThread;
-                ManualResetEventSlim reset = wrap.GetWorkerResetEvent(inWrapIndex);
-                do
-                {
-                    reset.Wait();
-                    reset.Reset();
-                    self.Priority = ThreadPriority.Highest;
-                    ExecuteTaskPackage(wrap.GetWorkerTaskQueue(inWrapIndex));
-                    _ = Interlocked.Decrement(ref RemainThreadCount);
-                    self.Priority = _AfterWork;
-                } while (true);
-            }
-
-            private static unsafe void MatrixParallelScheduler([AllowNull] object obj)
-            {
-                Handle handle = (obj as Handle)!;
-                MatrixParallelWrap wrap = handle.Wrap;
-                int inWrapIndex = handle.InWrapIndex;
-                Queue<IntPtr> taskQueue = wrap.GetWorkerTaskQueue(inWrapIndex);
-                ManualResetEventSlim resetEvent = wrap.SchedulerReset;
-                Thread self = Thread.CurrentThread;
-                do
-                {
-                    resetEvent.Wait();
-                    resetEvent.Reset();
-                    self.Priority = ThreadPriority.Highest;
-
-                    // TODO task publication to worker in wrap
-                    wrap.SetAllWorker();
-                    ExecuteTaskPackage(taskQueue);
-                    _ = Interlocked.Decrement(ref RemainThreadCount);
-                    self.Priority = _AfterWork;
-                } while (true);
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public static unsafe void ExecuteTaskPackage(Queue<IntPtr> taskQueue)
-            {
-                while (taskQueue.TryDequeue(out nint handle))
-                {
-                    MatrixParallelTaskPackageNonGeneric* p = (MatrixParallelTaskPackageNonGeneric*)handle;
-                    switch (p->ComputeMode)
-                    {
-                        case 1:
-                            Compute_Unary(p);
-                            break;
-                        case 2:
-                            Compute_UnaryScalar(p);
-                            break;
-                        case 3:
-                            Compute_Binary(p);
-                            break;
-                        case 4:
-                            Compute_BinaryScalar(p);
-                            break;
-                        default:
-                            break;
-                    }
-                }
-            }
-
-            #endregion
-        }
-
-        public enum SleepMode
-        {
-
-            QuickWakeup = ThreadPriority.AboveNormal,
-            NormalSleep = ThreadPriority.BelowNormal,
-            DeepSleep = ThreadPriority.Lowest,
-
-        }
-
         #region wrap field
 
         private static CapacityFixedStack<MatrixParallelWrap> Wraps;
@@ -370,6 +201,7 @@ namespace StgSharp.Mathematics.Numeric
 #else
             Monitor.Enter(_lock);
 #endif
+
             return Wraps;
         }
 

StgSharp.Mathematics/Numeric/MatrixParallelPool.cs

@@ -0,0 +1,74 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="MatrixParallelPool"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using StgSharp.Collections;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.Mathematics.Numeric
+{
+    public class MatrixParallelPool
+    {
+
+        private MatrixParallelPool()
+        {
+            throw new NotSupportedException();
+        }
+
+        internal MatrixParallelPool(int threadCount, ReadOnlySpan<MatrixParallelWrap> wrapSpan)
+        {
+            SpareWrap = CapacityFixedStackBuilder.Create(wrapSpan);
+            SingleThreadAllocation = new CapacityFixedBag<MatrixParallelWrap>(wrapSpan.Length);
+            SpareThreads = threadCount;
+        }
+
+        public int SpareThreads { get; set; }
+
+        internal CapacityFixedBag<MatrixParallelWrap> SingleThreadAllocation { get; set; }
+
+        internal CapacityFixedStack<MatrixParallelWrap> SpareWrap { get; set; }
+
+        public bool TryRequest(out )
+
+
+        public bool TryRequestWrap(out MatrixParallelWrap? wrap)
+        {
+            if (SpareWrap.TryPop(out wrap))
+            {
+                SpareThreads -= wrap.WrapTaskCapacity;
+                return true;
+            } else
+            {
+                return false;
+            }
+        }
+
+    }
+}

StgSharp.Mathematics/Numeric/MatrixParallelTask.cs

@@ -1,30 +1,27 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="MatrixParallel.Task.cs"
+// file="MatrixParallelTask"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
-// Permission is hereby granted, free of charge, to any person 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
-// obtaining a copy of this software and associated documentation 
+// of this software and associated documentation files (the "Software"), to deal
-// files (the “Software”), to deal in the Software without restriction, 
+// in the Software without restriction, including without limitation the rights
-// including without limitation the rights to use, copy, modify, merge,
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// publish, distribute, sublicense, and/or sell copies of the Software, 
+// copies of the Software, and to permit persons to whom the Software is
-// and to permit persons to whom the Software is furnished to do so, 
+// furnished to do so, subject to the following conditions:
-// subject to the following conditions:
+//
-//     
+// The above copyright notice and this permission notice shall be included in all
-// The above copyright notice and 
+// copies or substantial portions of the Software.
-// this permission notice shall be included in all copies 
+//
-// or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-//     
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// THE SOFTWARE IS PROVIDED “AS IS”, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
+// SOFTWARE.
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 
-// ARISING FROM, OUT OF OR IN CONNECTION WITH 
-// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------

StgSharp.Mathematics/Numeric/MatrixParallelWrap.cs

@@ -0,0 +1,209 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="MatrixParallelWrap"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.Mathematics.Numeric
+{
+    public unsafe class MatrixParallelWrap
+    {
+
+        private readonly Queue<IntPtr>[] _cache;
+        private readonly Handle[] _resetHandle;
+        private readonly Thread[] _threads;
+
+        public MatrixParallelWrap(int id, int wrapSize)
+        {
+            ID = id;
+            _cache = new Queue<IntPtr>[wrapSize];
+            _threads = new Thread[wrapSize];
+            for (int i = 0; i < wrapSize; i++)
+            {
+                _cache[i] = new Queue<IntPtr>(8);
+                _threads[i] = new Thread(MatrixParallelWorker)
+                {
+                    IsBackground = true,
+                    Priority = ThreadPriority.BelowNormal,
+                    Name = $"MPT#{ID}_{i}"
+                };
+                _threads[i].Start(new Handle(i, this));
+            }
+        }
+
+        public int ID { get; init; }
+
+        public static SleepMode AfterWork
+        {
+            get;
+            set => Interlocked.Exchange(ref Unsafe.As<SleepMode, int>(ref field), (int)value);
+        }
+
+        internal ManualResetEventSlim SchedulerReset { get; } = new(false);
+
+        public Span<IntPtr> EnqueuePackage(Span<IntPtr> packageSpan)
+        {
+            int idx = 0;
+            for (; idx < _cache.Length; idx++) {
+                _cache[idx].Enqueue(packageSpan[idx]);
+            }
+            return packageSpan[idx..];
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Queue<IntPtr> GetWorkerTaskQueue(int idInWrap)
+        {
+            return _cache[idInWrap];
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void SetAllWorker()
+        {
+            foreach (Handle item in _resetHandle) {
+                item.ResetEvent.Set();
+            }
+        }
+
+        public class Handle(int inWrapIndex, MatrixParallelWrap wrap)
+        {
+
+            public int InWrapIndex { get; } = inWrapIndex;
+
+            /// <summary>
+            ///   Role of the thread in the wrap. 1 is Scheduler, 0 are Workers. 2 is leader, beyond
+            ///   wrap's control.
+            /// </summary>
+            public int Role { get; internal set; }
+
+            internal ManualResetEventSlim ResetEvent { get; } = new(false);
+
+            internal MatrixParallelWrap Wrap { get; } = wrap;
+
+        }
+
+        #region wrap task
+
+        internal MatrixParallelTaskPackageNonGeneric* WrapTask { get; set; }
+
+        internal int WrapTaskSliceCount { get; set; }
+
+        internal int WrapTaskCapacity
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get => _threads.Length;
+        }
+
+            #endregion
+
+
+        #region ThreadOperation
+
+        private static unsafe void MatrixParallelWorker([AllowNull] object obj)
+        {
+            Handle handle = (obj as Handle)!;
+            MatrixParallelWrap wrap = handle.Wrap;
+            int inWrapIndex = handle.InWrapIndex;
+            Thread self = Thread.CurrentThread;
+            ManualResetEventSlim reset = handle.ResetEvent;
+            do
+            {
+                reset.Wait();
+                reset.Reset();
+                self.Priority = ThreadPriority.Highest;
+                ExecuteTaskPackage(wrap.GetWorkerTaskQueue(inWrapIndex));
+
+                // _ = Interlocked.Decrement(ref RemainThreadCount);
+                self.Priority = (ThreadPriority)AfterWork;
+            } while (true);
+        }
+
+        private static unsafe void MatrixParallelScheduler([AllowNull] object obj)
+        {
+            Handle handle = (obj as Handle)!;
+            MatrixParallelWrap wrap = handle.Wrap;
+            int inWrapIndex = handle.InWrapIndex;
+            Queue<IntPtr> taskQueue = wrap.GetWorkerTaskQueue(inWrapIndex);
+            ManualResetEventSlim resetEvent = wrap.SchedulerReset;
+            Thread self = Thread.CurrentThread;
+            do
+            {
+                resetEvent.Wait();
+                resetEvent.Reset();
+                self.Priority = ThreadPriority.Highest;
+
+                // TODO task publication to worker in wrap
+                wrap.SetAllWorker();
+                ExecuteTaskPackage(taskQueue);
+
+                // _ = Interlocked.Decrement(ref RemainThreadCount);
+                self.Priority = (ThreadPriority)AfterWork;
+            } while (true);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static unsafe void ExecuteTaskPackage(Queue<IntPtr> taskQueue)
+        {
+            while (taskQueue.TryDequeue(out nint handle))
+            {
+                MatrixParallelTaskPackageNonGeneric* p = (MatrixParallelTaskPackageNonGeneric*)handle;
+                switch (p->ComputeMode)
+                {
+                    case 1:
+                        MatrixComputeModel.Compute_Unary(p);
+                        break;
+                    case 2:
+                        MatrixComputeModel.Compute_UnaryScalar(p);
+                        break;
+                    case 3:
+                        MatrixComputeModel.Compute_Binary(p);
+                        break;
+                    case 4:
+                        MatrixComputeModel.Compute_BinaryScalar(p);
+                        break;
+                    default:
+                        break;
+                }
+            }
+        }
+
+        #endregion
+    }
+
+    public enum SleepMode : int
+    {
+
+        QuickWakeup = ThreadPriority.AboveNormal,
+        NormalSleep = ThreadPriority.BelowNormal,
+        DeepSleep = ThreadPriority.Lowest,
+
+    }
+}