matrix parallel start to use two different policy

StgSharp.Common/Collections/LinearBag.cs

@@ -151,10 +151,11 @@ namespace StgSharp.Collections
         ///   Removes the first occurrence of value (unordered). Does nothing if not found.
         /// </summary>
         public void Remove(
-                    T value
+                    T value,
+                    EqualityComparer<T> comparer = default
         )
         {
-            EqualityComparer<T> comparer = EqualityComparer<T>.Default;
+            comparer ??= EqualityComparer<T>.Default;
             for (int i = 0; i < Count; i++)
             {
                 if (comparer.Equals(_items[i], value))

StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs

@@ -39,7 +39,7 @@ namespace StgSharp.HighPerformance.Memory
         private struct PredictionArray
         {
 
-            private short PredictionCount;
+            private int PredictionCount;
 
         }
 
@@ -47,7 +47,7 @@ namespace StgSharp.HighPerformance.Memory
         private struct MapArray
         {
 
-            private short MapCount;
+            private int MapCount;
 
         }
 

StgSharp.Common/HighPerformance/Memory/L4.Evict.cs

@@ -25,6 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
+using StgSharp.Collections;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -56,7 +57,29 @@ namespace StgSharp.HighPerformance.Memory
         )
         {
             count = (count < 0) ? 0 : count;
-            return _intrinsic.ScanEvict((short*)_predictCount, (short*)_mapCount, count, this);
+            int* pPtr = (int*)_predictCount, mPtr = (int*)_mapCount;
+
+
+            int remain = count;
+            ConcurrentFlexibleArray<IL4Predict> predictors = _predictors;
+            predictors.RequestSpanOperation();
+            Span<IL4Predict> pSpan = predictors.AsSpan();
+            try
+            {
+                for (int i = 0; i < CacheLineCount; i++)
+                {
+                    if (pPtr[i] > mPtr[i])
+                    {
+                        continue;
+                    }
+                    remain -= TryEvictAt(i, pSpan) ? 1 : 0;
+                }
+            }
+            finally
+            {
+                predictors.ReleaseSpanOperation();
+            }
+            return remain;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -85,7 +108,7 @@ namespace StgSharp.HighPerformance.Memory
                 predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
                 RecycleLine(index);
                 (*_mapCount)[index] = 0;
-                (*_predictCount)[index] = short.MaxValue;
+                (*_predictCount)[index] = int.MaxValue;
                 return true;
             }
             return false;

StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs

@@ -1,117 +0,0 @@
-//-----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-// file="L4.IntrinsicKernel"
-// Project: StgSharp
-// AuthorGroup: Nitload
-// Copyright (c) Nitload. All rights reserved.
-//     
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-//     
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-using StgSharp.Collections;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Numerics;
-using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.X86;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace StgSharp.HighPerformance.Memory
-{
-    public unsafe partial class L4
-    {
-
-        private interface IIntrinsicKernel
-        {
-
-            static IIntrinsicKernel Instance { get; } = Create();
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public int ScanEvict(
-                       short* pPtr,
-                       short* mPtr,
-                       int count,
-                       L4 owner
-            );
-
-            private static IIntrinsicKernel Create()
-            {
-                if (Sse42.IsSupported)
-                {
-                    return new SseKernel();
-                } else
-                {
-                    throw new PlatformNotSupportedException("No supported intrinsic kernel found for current platform.");
-                }
-            }
-
-        }
-
-        private readonly struct SseKernel : IIntrinsicKernel
-        {
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public readonly int ScanEvict(
-                                short* pPtr,
-                                short* mPtr,
-                                int count,
-                                L4 owner
-            )
-            {
-                int remain = count;
-                ConcurrentFlexibleArray<IL4Predict> predictors = owner._predictors;
-                predictors.RequestSpanOperation();
-                Span<IL4Predict> pSpan = predictors.AsSpan();
-                try
-                {
-                    for (int i = 0; i < CacheLineCount; i += 8)
-                    {
-                        Vector128<short> p = Sse2.LoadVector128(pPtr + i);
-                        Vector128<short> m = Sse2.LoadVector128(mPtr + i);
-                        Vector128<short> great = Sse2.CompareGreaterThan(p, m);
-                        Vector128<short> le = Sse42.Xor(great, Vector128.Create<short>(-1));
-
-                        int mask = Sse42.MoveMask(le.AsByte());
-                        while ((mask != 0) && (remain > 0))
-                        {
-                            int bit = BitOperations.TrailingZeroCount(mask);
-                            int lane = bit >> 1;
-
-                            if (owner.TryEvictAt(i + lane, pSpan)) {
-                                remain--;
-                            }
-                            mask &= mask - 1;
-                            mask &= mask - 1;
-                        }
-                    }
-                }
-                finally
-                {
-                    predictors.ReleaseSpanOperation();
-                }
-                return remain;
-            }
-
-        }
-
-    }
-}

StgSharp.Common/HighPerformance/Memory/L4.cs

@@ -190,7 +190,7 @@ namespace StgSharp.HighPerformance.Memory
                         (*_predictCount)[entry->Id] = 0;
                         _ = Interlocked.Increment(ref _activeCount);
                         _ = Interlocked.Decrement(ref _aheadCount);
-                        _ = Interlocked.Decrement(ref entry->RemainingPredictionCount);
+                        _ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]);
                         return new L4CacheLine(entry->Address, ref _activeCount, ref entry->RefCount);
                     } else
                     {

StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Double.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="MatrixEnumerator"
+// file="MatrixCompute.Sequential"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -26,73 +26,21 @@
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 using StgSharp.HighPerformance.Memory;
+using StgSharp.Mathematics.Numeric.Runtime;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
-namespace StgSharp.Mathematics.Numeric.Runtime
+using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
+
+namespace StgSharp.Mathematics.Numeric
 {
-    public enum PredictPolicy
+    public static unsafe partial class MatrixCompute
     {
 
-        Transparent = 0,
-        BufferSequential = 1,
-        SquareRowMajor = 2,
-        SquareColMajor = 3,
-
-    }
-
-    public unsafe interface IMatrixCachePredictor<TSelf> : IL4Predict
-        where TSelf : class, IMatrixCachePredictor<TSelf>
-    {
-
-        public static abstract PredictPolicy Policy { get; set; }
-
-    }
-
-    public sealed unsafe class BufferSequentialPredictor : IMatrixCachePredictor<BufferSequentialPredictor>
-    {
-
-        private readonly MatrixParallelTask* origin;
-        private readonly int _kernelSize;
-
-        public static PredictPolicy Policy { get; set; } = PredictPolicy.SquareRowMajor;
-
-        public bool IsSamePolicy(
-                    uint policy,
-                    IL4Predict predict
-        )
-        {
-            throw new NotImplementedException();
-        }
-
-        public PredictResult Predict(
-                             Span<CacheLineDescription> node,
-                             out int count
-        )
-        {
-            throw new NotImplementedException();
-        }
-
-        public void Prefetch(
-                    nuint origin,
-                    uint policy,
-                    Span<byte> cache
-        )
-        {
-            throw new NotImplementedException();
-        }
-
-        public void WriteBack(
-                    nuint origin,
-                    uint policy,
-                    ReadOnlySpan<byte> cache
-        )
-        {
-            throw new NotImplementedException();
-        }
 
     }
 }

StgSharp.Mathematics/Numeric/Linear/MatrixCompute.Single.cs

@@ -1,6 +1,6 @@
 //-----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-// file="MatrixCompute.Sequential"
+// file="MatrixCompute.Single"
 // Project: StgSharp
 // AuthorGroup: Nitload
 // Copyright (c) Nitload. All rights reserved.
@@ -25,18 +25,16 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
-using StgSharp.HighPerformance.Memory;
 using StgSharp.Mathematics.Numeric.Runtime;
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
 using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
 
-namespace StgSharp.Mathematics.Numeric.Linear
+namespace StgSharp.Mathematics.Numeric
 {
     public static unsafe partial class MatrixCompute
     {
@@ -78,33 +76,5 @@ namespace StgSharp.Mathematics.Numeric.Linear
             SequentialOperation(t, count);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static void SequentialOperation(
-                            MatrixParallelTask* task,
-                            long count
-        )
-        {
-            MatrixElementType elementType = task->ElementType;
-            MatrixParallelThread[] pool;
-            MkProc op = (MkProc)NumericalModule.GlobalContext.
-                mat_mk[elementType.IntrinsicNode].
-                Get(task->ComputeHandle);
-            if ((count <= 256) || !MatrixParallel.IsParallelMeaningful)
-            {
-                op(task);
-                return;
-            } else if (count <= 1024)
-            {
-                pool = MatrixParallel.LeadParallel(1);
-                op(task);
-                MatrixParallel.ReturnParallelResources(ref pool);
-                return;
-            }
-            pool = MatrixParallel.LeadParallel(count / 1024);
-            MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask<BufferSequentialPredictor>(pool, task);
-            taskGroup.LaunchParallel(SleepMode.DeepSleep);
-            taskGroup.Dispose();
-        }
-
     }
 }

StgSharp.Mathematics/Numeric/Linear/MatrixCompute.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="MatrixCompute"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -28,6 +28,7 @@
 using StgSharp.HighPerformance.Memory;
 using StgSharp.Internal;
 using StgSharp.Mathematics;
+using StgSharp.Mathematics.Numeric.Runtime;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -36,9 +37,11 @@ using System.Runtime.CompilerServices;
 using System.Text;
 using System.Threading.Tasks;
 
+using unsafe MkProc = delegate* unmanaged[Cdecl]<StgSharp.Mathematics.Numeric.Runtime.MatrixParallelTask*, void>;
+
 namespace StgSharp.Mathematics.Numeric
 {
-    public static partial class MatrixCompute
+    public static unsafe partial class MatrixCompute
     {
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -49,7 +52,28 @@ namespace StgSharp.Mathematics.Numeric
                                              int y
         ) where T : unmanaged, INumber<T>
         {
-            return (MatrixKernel*)(buffer + (colLength * x + y));
+            return (MatrixKernel*)(buffer + ((colLength * x) + y));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static void SequentialOperation(
+                            MatrixParallelTask* task,
+                            long count
+        )
+        {
+            MatrixElementType elementType = task->ElementType;
+            MatrixParallelThread[] pool;
+            MkProc op = (MkProc)NumericalModule.GlobalContext.
+                mat_mk[elementType.IntrinsicNode].
+                Get(task->ComputeHandle);
+            if ((count <= 256) || !MatrixParallel.IsParallelMeaningful)
+            {
+                op(task);
+                return;
+            }
+            MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferSequentialQueue>(task);
+            taskWrap.LaunchParallel(SleepMode.DeepSleep);
+            taskWrap.Dispose();
         }
 
     }

StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs

@@ -50,6 +50,7 @@ namespace StgSharp.Mathematics.Numeric.Runtime
 
         RowMajor = 4,
         ColMajor = 5,
+        Sequential = 6,
 
         #endregion
 

StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Numa.cs

@@ -42,8 +42,8 @@ namespace StgSharp.Mathematics.Numeric
         private static NumaMeasurement[] _numaMeasurements;
 
         private static int[] _numaPayload;
-        private static LinearBag<MatrixParallelThread>[] _numaSyncHeavy;
+        private static MatrixParallelThread[][] _numaSyncHeavy;
-        private static LinearBag<MatrixParallelThread>[] _numaWaveFront;
+        private static MatrixParallelThread[][] _numaWaveFront;
         private static MatrixParallelThread[] _threads;
 
         private static void UpdateThreadPayload()

StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Publish.cs

@@ -49,9 +49,9 @@ namespace StgSharp.Mathematics.Numeric
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
 
-        internal static unsafe MatrixParallelWrap ScheduleTask<TTask>(
+        internal static unsafe MatrixParallelWrap ScheduleTask<TTaskQueue>(
                                                   MatrixParallelTask* package
-        ) where TTask : IL4Predict
+        ) where TTaskQueue : IMatrixParallelQueue<TTaskQueue>
         {
             /*
             int primCount = package->PrimCount;
@@ -73,9 +73,10 @@ namespace StgSharp.Mathematics.Numeric
                 ulong total = (ulong)_numaWaveFront[i].Length;
                 NumaMeasurement measure = _numaMeasurements[i];
                 ulong numaPayload = measure.ActiveTasks;
-                uint numaActive = measure.ActiveThreadCount;
+
+                // uint numaActive = measure.ActiveThreadCount;
                 long curPayload = ((long)total) - (((long)(numaPayload + additionalPayload + 63)) / 64);
-                if (curPayload > payload)
+                if (curPayload < payload)
                 {
                     payload = curPayload;
                     payloadIndex = i;

StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.cs

@@ -84,7 +84,7 @@ namespace StgSharp.Mathematics.Numeric
             _retiredThreads = new(RESERVED_THREAD_COUNT);
 
             // init thread on all logical cores
-            MatrixParallelThread[][] numaWaveOrigin = Numa.CreateForeachThread<MatrixParallelThread>(
+            _numaWaveFront = Numa.CreateForeachThread<MatrixParallelThread>(
                core => {
                    MatrixParallelThread thread = new(core);
                    SpinWait.SpinUntil(() => thread.State == MatrixThreadState.Ready);
@@ -108,15 +108,15 @@ namespace StgSharp.Mathematics.Numeric
             // retire threads
             if (ContextCount == 1)
             {
-                MatrixParallelThread[] threadArr = numaWaveOrigin[0];
+                MatrixParallelThread[] threadArr = _numaWaveFront[0];
                 int retireCount = Math.Min(RESERVED_THREAD_COUNT, Math.Max(0, threadArr.Length));
                 for (int i = 1; i <= retireCount; i++) {
                     RetireThread(threadArr[^i]);
                 }
-                _numaWaveFront[0] = [.. threadArr[0..^retireCount]];
+                _numaWaveFront[0] = threadArr[0..^retireCount];
             } else
             {
-                RetireWhenMultipleSIMDID(numaWaveOrigin);
+                RetireWhenMultipleSIMDID(_numaWaveFront);
             }
             _meaningful = _threads.Length > 1;
 
@@ -151,7 +151,6 @@ namespace StgSharp.Mathematics.Numeric
             ) => CompareSIMDID(left.Simd, right.Simd));
             int retireCount = Math.Min(RESERVED_THREAD_COUNT, Math.Max(0, allThreads.Length));
 
-            // HashSet<MatrixParallelThread> retiredSet = [];
             MatrixParallelThread[] retireArr = new MatrixParallelThread[retireCount];
             for (int i = 0; i < retireCount; i++)
             {
@@ -183,7 +182,7 @@ namespace StgSharp.Mathematics.Numeric
                     }
                 }
 
-                _numaWaveFront[numa] = [.. active];
+                _numaWaveFront[numa] = active;
             }
 
             _threads = allThreads[retireCount..];

StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs

@@ -39,18 +39,96 @@ using System.Threading.Tasks;
 
 namespace StgSharp.Mathematics.Numeric
 {
-    internal unsafe class MatrixParallelQueue<TPredictor>
+    internal unsafe interface IMatrixParallelQueue<TSelf> where TSelf : IMatrixParallelQueue<TSelf>
-        where TPredictor : class, IMatrixCachePredictor<TPredictor>
     {
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        static abstract bool NeedNumaBuffer { get; }
-        public bool Steal(
+
-                    MatrixParallelTask* task
+        static abstract TSelf Create(
+                              MatrixParallelTask* baseTask,
+                              L4 buffer
+        );
+
+        void Return(
+             MatrixParallelTask* task
+        );
+
+        bool Steal(
+             MatrixParallelTask* task
+        );
+
+    }
+
+    internal sealed unsafe class BufferSequentialQueue : IMatrixParallelQueue<BufferSequentialQueue>
+    {
+
+        private MatrixParallelTask* baseTask;
+
+        private BufferSequentialQueue() { }
+
+        private BufferSequentialQueue(
+                MatrixParallelTask* baseTask
+        )
+        {
+            this.baseTask = baseTask;
+        }
+
+        public static bool NeedNumaBuffer => throw new NotImplementedException();
+
+        public static unsafe BufferSequentialQueue Create(
+                                                   MatrixParallelTask* baseTask,
+                                                   L4 buffer
+        )
+        {
+            return new BufferSequentialQueue(baseTask);
+        }
+
+        public unsafe void Return(
+                           MatrixParallelTask* task
+        ) { }
+
+        public unsafe bool Steal(
+                           MatrixParallelTask* task
         )
         {
-            // TODO 使用提取的内容填充task
             throw new NotImplementedException();
         }
 
+        private class Predictor : IL4Predict
+        {
+
+            public PredictResult Predict(
+                                 Span<CacheLineDescription> node,
+                                 out int count
+            )
+            {
+                throw new NotImplementedException();
+            }
+
+            public void Prefetch(
+                        nuint origin,
+                        uint policy,
+                        Span<byte> cache
+            )
+            {
+                throw new NotImplementedException();
+            }
+
+            public void WriteBack(
+                        nuint origin,
+                        uint policy,
+                        ReadOnlySpan<byte> cache
+            )
+            {
+                MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
+                if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
+                {
+                    byte* span = (byte*) origin;
+                    cache.CopyTo(new Span<byte>(span, cache.Length));
+                }
+            }
+
+        }
+
     }
 }

StgSharp.Native/sn-head/StgSharpNative.h

@@ -23,6 +23,9 @@
 #define cs_char uint16_t
 #define sn_null_assert(ptr) assert(ptr != NULL)
 
+#ifndef SN_
+#endif
+
 #define ALIGN(simdVec) _mm_loadu_ps(&simdVec)
 #define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec)
 #define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec)

StgSharp.Native/sn-head/sn_intrinsic.context.x86.h

@@ -6,9 +6,9 @@
 #include "sn_intrinsic.std.h"
 
 #ifndef SN_X86_TARGET_ATTR_sse
-#define SN_X86_TARGET_ATTR_sse SN_CLANG_TARGET_ATTR("sse2")
+#define SN_X86_TARGET_ATTR_sse SN_CLANG_TARGET_ATTR("sse42")
 #define SN_X86_TARGET_ATTR_avx SN_CLANG_TARGET_ATTR("avx")
-#define SN_X86_TARGET_ATTR_avx_fma SN_CLANG_TARGET_ATTR("avx,fma")
+#define SN_X86_TARGET_ATTR_avx_fma SN_CLANG_TARGET_ATTR("avx,avx2,fma")
 #define SN_X86_TARGET_ATTR_512 SN_CLANG_TARGET_ATTR("avx512f,avx512vl")
 #endif