after merge

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs

@@ -26,6 +26,7 @@
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 using System;
+using System.IO;
 using System.Numerics;
 using System.Reflection.Metadata;
 using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
@@ -39,49 +40,40 @@ namespace StgSharp.HighPerformance.Memory
                                   nuint size
         )
         {
-            if (size > _maxSize) {
+            if (size > _maxAllocSize) {
                 throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
             }
-
+            /*
             int level = GetLevelFromSize(size);
-            int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
+            int segmentIndex = GetSegmentFromLevel(size, level);
 
             int i = level, s = segmentIndex;
-            Entry* handle;
+            /**/
-            while (!TryPopLevelForAllocation(i, s, out handle))
+
-            {
+            Entry* entry = null;
-                if (++s >= 3)
+            nuint requestedOffset = GetAllocSize(size);
-                {
+            (int level, int segment) = GetIndexPairOnce(size);
-                    s = 0;
+            int index = (3 * level) + segment;
+            int i = index;
+            while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
                 i++;
             }
-                if (i > MaxLevel)
+            if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
-                {
-                    if (TryPopLevelForAllocation(10, 1, out handle))
-                    {
-                        break;
-                    } else if (TryAllocateNew32MBBlock(out handle))
-                    {
-                        break;
-                    }
                 throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
             }
-            }
+            if (entry != null)
-
-            if (handle != null)
             {
-                int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
+                if (requestedOffset > entry->Size) {
-                if (requestedOffset > (int)handle->Size) {
                     throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
                 }
-                nuint remainder = handle->Size - ((nuint)requestedOffset);
+                nuint remainder = entry->Size - requestedOffset;
-                SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
+                SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
-                handle->State = EntryState.Alloc;
+                entry->State = EntryState.Alloc;
                 if (remainder > 0) {
-                    SliceMemory(handle->NextNear, remainder);
+                    OrganizeMemory(entry->NextNear, remainder);
                 }
             }
-            return new(handle, size);
+            return new(entry, size);
         }
 
         public bool TryAlloc(
@@ -89,49 +81,40 @@ namespace StgSharp.HighPerformance.Memory
                     out HlsfHandle handle
         )
         {
-            if (size > _maxSize)
+            if (size > _maxAllocSize)
             {
                 handle = default;
                 return false;
             }
-
+            /*
             int level = GetLevelFromSize(size);
-            int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
+            int segmentIndex = GetSegmentFromLevel(size, level);
 
             int i = level, s = segmentIndex;
-            Entry* entry;
+            /**/
-            while (!TryPopLevelForAllocation(i, s, out entry))
+
-            {
+            Entry* entry = null;
-                if (++s >= 3)
+            nuint requestedOffset = GetAllocSize(size);
-                {
+            (int level, int segment) = GetIndexPairOnce(size);
-                    s = 0;
+            int index = (3 * level) + segment;
+            int i = index;
+            while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
                 i++;
             }
-                if (i > MaxLevel)
+            if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
-                {
+                throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
-                    if (TryPopLevelForAllocation(10, 1, out entry))
-                    {
-                        break;
-                    } else if (TryAllocateNew32MBBlock(out entry))
-                    {
-                        break;
-                    }
-                    handle = default;
-                    return false;
-                }
             }
 
             if (entry != null)
             {
-                int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
+                if (requestedOffset > entry->Size) {
-                if (requestedOffset > (int)entry->Size) {
                     throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
                 }
-                nuint remainder = entry->Size - ((nuint)requestedOffset);
+                nuint remainder = entry->Size - requestedOffset;
                 SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
                 entry->State = EntryState.Alloc;
                 if (remainder > 0) {
-                    SliceMemory(entry->NextNear, remainder);
+                    OrganizeMemory(entry->NextNear, remainder);
                 }
             }
             handle = new(entry, size);
@@ -139,16 +122,16 @@ namespace StgSharp.HighPerformance.Memory
         }
 
         /// <summary>
-        ///   Slice the memory block into smaller blocks and add them to the appropriate buckets
+        ///   Slice the memory block into smaller blocks and add them to the appropriate buckets.
         ///   Notice: this method is not responsible for freeing the original BucketNode
         /// </summary>
         /// <param name="e">
-        ///
+        ///   The next entry nearby the part to slice
         /// </param>
         /// <param name="sizeToSlice">
         ///
         /// </param>
-        private void SliceMemory(
+        private void OrganizeMemory(
                      Entry* e,
                      nuint sizeToSlice
         )
@@ -159,97 +142,22 @@ namespace StgSharp.HighPerformance.Memory
             nuint end = e->Position - ((nuint)m_Buffer);                      // end offset of buffer to slice
 
             int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
-            i = int.Min(10, i);
-            for (; i < _levelSizeArray.Length; i++)
-            {
-                uint size = _levelSizeArray[i];
-                nuint upper = size * 4;
-                nuint edge = (((offset + upper) - 1U) / upper) * upper;
-                int count = (int)((nuint.Min(edge, end) - offset) / size);
-                if (count > 0)
-                {
-                    nuint newSize = ((nuint)count) * size;
-
-                    Entry* newEntry = (Entry*)_entries.Allocate();
-                    InsertEntryBefore(newEntry, next);
-                    SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
-                    newEntry->State = EntryState.Empty;
-
-                    BucketNode* b = (BucketNode*)newEntry->Position;
-                    b->EntryRef = newEntry;
-                    PushBucketToLevel(i, count - 1, b);
-                    offset += newSize;
-                    remain -= newSize;
-                }
-                if (remain <= 0) {
-                    return;
-                }
-            }
-
-            if (i > MaxLevel)
-            {
-                uint size = _levelSizeArray[^1];
-                nuint upper = size * 4;
-                nuint edge = (end / upper) * upper;
-                int count = (int)((nuint.Min(edge, end) - offset) / upper);
-                if (count > 0)
-                {
-                    nuint newSize = ((nuint)count) * upper;
-
-                    Entry* newEntry = (Entry*)_entries.Allocate();
-                    InsertEntryBefore(newEntry, next);
-                    SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
-                    newEntry->State = EntryState.Empty;
-
-                    BucketNode* b = (BucketNode*)newEntry->Position;
-                    b->EntryRef = newEntry;
-                    PushBucketToLevel(i, count - 1, b);
-                    offset += newSize;
-                    remain -= newSize;
-                    if (remain <= 0) {
-                        return;
-                    }
-                }
-            }
-
-            for (i--; i >= 0; i--)
-            {
-                uint size = _levelSizeArray[i];
-                nuint edge = (end / size) * size;
-                int count = (int)((nuint.Min(edge, end) - offset) / size);
-                if (count > 0)
-                {
-                    nuint newSize = ((nuint)count) * size;
-
-                    Entry* newEntry = (Entry*)_entries.Allocate();
-                    InsertEntryBefore(newEntry, next);
-                    SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
-                    newEntry->State = EntryState.Empty;
-
-                    BucketNode* b = (BucketNode*)newEntry->Position;
-                    b->EntryRef = newEntry;
-                    b->NextLevel = null;
-                    b->PreviousLevel = null;
-                    PushBucketToLevel(i, count - 1, b);
-                    offset += newSize;
-                    remain -= newSize;
-                }
-                if (remain <= 0) {
-                    return;
-                }
-            }
         }
 
         /// <summary>
         ///   Try to allocate a new 16MB block when all buckets are empty
         /// </summary>
+        /// <param name="size">
+        ///   Size of memory block to be allocated. Notice: Size is assumed as times of 64.
+        /// </param>
         /// <param name="handle">
         ///   Output handle to the allocated Entry
         /// </param>
         /// <returns>
         ///   True if allocation succeeded
         /// </returns>
-        private bool TryAllocateNew32MBBlock(
+        private bool TryAllocateNewBlock(
+                     nuint size,
                      out Entry* handle
         )
         {
@@ -260,7 +168,7 @@ namespace StgSharp.HighPerformance.Memory
                 handle = null;
                 return false;
             }
-            nuint size = (remain > 32 * 1024 * 1024L) ? (32 * 1024 * 1024U) : remain;
+            size = nuint.Min(size, remain);
 
 
             // build new entry
@@ -278,6 +186,23 @@ namespace StgSharp.HighPerformance.Memory
             return true;
         }
 
+        /// <summary>
+        ///   Helper method for allocation - converts BucketNode to Entry
+        /// </summary>
+        private bool TryPopLevelForAllocation(
+                     int bucketIndex,
+                     out Entry* entry
+        )
+        {
+            if (TryPopBucket(bucketIndex, out BucketNode* bucketNode))
+            {
+                entry = bucketNode->EntryRef;
+                return entry != null;
+            }
+            entry = null;
+            return false;
+        }
+
         /// <summary>
         ///   Helper method for allocation - converts BucketNode to Entry
         /// </summary>

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs

@@ -35,6 +35,7 @@ namespace StgSharp.HighPerformance.Memory
     public unsafe partial class HybridLayerSegregatedFitAllocator
     {
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void PushBucketToLevel(
                      int levelIndex,
                      int sizeIndex,
@@ -45,7 +46,16 @@ namespace StgSharp.HighPerformance.Memory
                 return;
             }
 
-            Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
+            int index = (3 * levelIndex) + sizeIndex;
+            PushNodeToBucket(index, node);
+        }
+
+        private void PushNodeToBucket(
+                     int index,
+                     BucketNode* node
+        )
+        {
+            index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
             if (_bucketHeads[index] is null)
             {
                 _bucketHeads[index] = node;
@@ -79,10 +89,11 @@ namespace StgSharp.HighPerformance.Memory
                      BucketNode* node
         )
         {
-            Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
+            int index = (3 * levelIndex) + sizeIndex;
+            index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
             if (_bucketHeads[index] == node)
             {
-                if (TryPopLevel(levelIndex, sizeIndex, out _)) {
+                if (TryPopBucket(index, out _)) {
                     return;
                 }
             }
@@ -102,14 +113,16 @@ namespace StgSharp.HighPerformance.Memory
             return;
         }
 
-        private bool TryPopLevel(
+        private bool TryPopBucket(
-                     int levelIndex,
+                     int index,
-                     int sizeIndex,
                      out BucketNode* node
         )
         {
-            Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
+            /*
-
+             * Attention:
+             * This method is UNSAFE. Index may exceed range of _bucketHeads.
+             * Be careful when calling this method.
+             */
             BucketNode* head = _bucketHeads[index];
 
             if (head == null)
@@ -126,24 +139,37 @@ namespace StgSharp.HighPerformance.Memory
                 _bucketHeads[index] = null;
                 node = head;
                 return true;
+            } else
+            {
+                next->PreviousLevel = null;
             }
             _bucketHeads[index] = head->NextLevel;
             node = head;
             node->IsInBucket = false;
             node->NextLevel = null;
-            if (next != null) {
-                next->PreviousLevel = null;
-            }
             return true;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool TryPopLevel(
+                     int levelIndex,
+                     int sizeIndex,
+                     out BucketNode* node
+        )
+        {
+            int index = (3 * levelIndex) + sizeIndex;
+            index = (index >= _bucketHeads.Length) ? (_bucketHeads.Length - 1) : index;
+            return TryPopBucket(index, out node);
+        }
+
         private bool TryRemoveFromBucket(
                      int levelIndex,
                      int sizeIndex,
                      BucketNode* node
         )
         {
-            Index index = (levelIndex > 9) ? (^1) : ((3 * levelIndex) + sizeIndex);
+            int index = (3 * levelIndex) + sizeIndex;
+            index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
             if (_bucketHeads[index] == node)
             {
                 bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
@@ -169,7 +195,7 @@ namespace StgSharp.HighPerformance.Memory
 
         /// <summary>
         ///   BucketNode structure - specialized node for bucket linked lists Stored in main
-        ///   allocation area (user-overwritable region)
+        ///   allocation area (user-writable region)
         /// </summary>
         [StructLayout(LayoutKind.Explicit)]
         internal struct BucketNode

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="HLSFAllocator.Debug"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -36,7 +36,8 @@ namespace StgSharp.HighPerformance.Memory
     public sealed class HLSFPositionChainBreakException : Exception
     {
 
-        public HLSFPositionChainBreakException() : base("Position chain of an HLSF allocator is broken") { }
+        public HLSFPositionChainBreakException()
+            : base("Position chain of an HLSF allocator is broken") { }
 
     }
 
@@ -45,7 +46,9 @@ namespace StgSharp.HighPerformance.Memory
 
         private object _lock = new object();
 
-        public void DisplayEntryLayout(bool singleStep)
+        public void DisplayEntryLayout(
+                    bool singleStep
+        )
         {
             lock (_lock)
             {
@@ -61,7 +64,10 @@ namespace StgSharp.HighPerformance.Memory
             }
         }
 
-        public void DumpPositionChain(bool singleStep = false, int maxCount = 4096)
+        public void DumpPositionChain(
+                    bool singleStep = false,
+                    int maxCount = 4096
+        )
         {
             lock (_lock)
             {
@@ -73,7 +79,7 @@ namespace StgSharp.HighPerformance.Memory
                         cur = cur->NextNear;
                     }
                     Console.WriteLine(EntryToString(_spareMemory));
-                    while (cur != null && cur != _spareMemory && cnt < maxCount)
+                    while ((cur != null) && (cur != _spareMemory) && (cnt < maxCount))
                     {
                         Console.WriteLine(EntryToString(cur));
                         cnt++;
@@ -112,7 +118,9 @@ namespace StgSharp.HighPerformance.Memory
             }
         }
 
-        private static string EntryToString(Entry* e)
+        private static string EntryToString(
+                              Entry* e
+        )
         {
             BucketNode* b = (BucketNode*)e->Position;
             return $"{(ulong)e->PreviousNear}<-prev- Position:{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -next->{(ulong)e->NextNear}";

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs

@@ -79,7 +79,7 @@ namespace StgSharp.HighPerformance.Memory
                     cur = next;
                 } while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
                 if (size > 0) {
-                    SliceMemory(cur, size);
+                    OrganizeMemory(cur, size);
                 }
             }
         }

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Security;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Text;
@@ -38,31 +39,51 @@ namespace StgSharp.HighPerformance.Memory
     public unsafe partial class HybridLayerSegregatedFitAllocator
     {
 
-        private const int MaxLevel = 9;
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-
+        private nuint GetAllocSize(
-        private uint[] _levelSizeArray = [
+                      nuint size
-            64U,
-            256U,
-            1024U,
-            4096U,
-            16384U,
-            65536U,
-            262144U,
-            1048576U,
-            4194304U,
-            16777216U
-        ];
-
-        public int AlignOfLevel(
-                   int bucketLevel
         )
+
         {
-            uint blockSize = _levelSizeArray[bucketLevel];
+            return (size + 63u) & (~63u);
-            return Math.Min((int)blockSize, _align);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int GetLevelFromSize(
+        private int GetBucketIndexOnce(
+                    nuint size
+        )
+        {
+            int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
+
+            // Calculate level: level = max(0, (log2Size - 6) / 2)
+            int level = Math.Max(0, (log2Size - 6) / 2);
+
+            // caller is responsible for ensuring level is within bounds
+            level = Math.Min(level, _maxLevel);
+            uint levelSize = 1u << (log2Size & (~1));
+            int segment = ((int)(size / levelSize)) - 1;
+            return (level * 3) + segment;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private (int level, int segment) GetIndexPairOnce(
+                                         nuint size
+        )
+        {
+            int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
+
+            // Calculate level: level = max(0, (log2Size - 6) / 2)
+            int level = Math.Max(0, (log2Size - 6) / 2);
+
+            // caller is responsible for ensuring level is within bounds
+            level = Math.Min(level, _maxLevel);
+            uint levelSize = 1u << (log2Size & (~1));
+            int segment = ((int)(size / levelSize)) - 1;
+            return (level, segment);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetLevelFromSize(
                     nuint size
         )
         {
@@ -83,27 +104,38 @@ namespace StgSharp.HighPerformance.Memory
             int level = Math.Max(0, (log2Size - 6) / 2);
 
             // caller is responsible for ensuring level is within bounds
-            return level;
+            return Math.Min(level, _maxLevel);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int GetLevelFromSize(
+        private static uint GetLevelSizeFromLevel(
-                           uint size
+                            int level
         )
         {
-            return GetLevelFromSize(size);
+            return 1u << (6 + (2 * level));
         }
 
-    }
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-
+        private static uint GetLevelSizeFromSize(
-    public enum CommonMemoryAlign : int
+                            nuint size
+        )
         {
+            int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
+            return 1u << (log2Size & (~1));
+        }
 
-        XMMLevel = 16,
+        /// <summary>
-        YMMLevel = 32,
+        ///   Determine segment index based on size and originLevel. Input level value is regarded
-        ZMMLevel = 64,
+        ///   safe.
-
+        /// </summary>
-        ArmNEONLevel = 16,
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetSegmentFromLevel(
+                    ulong size,
+                    int level
+        )
+        {
+            return ((int)(size / GetLevelSizeFromLevel(level))) - 1;
+        }
 
     }
 }

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs

@@ -37,6 +37,9 @@ namespace StgSharp.HighPerformance.Memory
         /// <summary>
         ///   Check if two Entries are adjacent in memory
         /// </summary>
+        /// <returns>
+        ///   True if two mem blocks are adjacent.
+        /// </returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static bool AssertAdjacentTo(
                             Entry* current,
@@ -48,36 +51,6 @@ namespace StgSharp.HighPerformance.Memory
                    (current->Position + current->Size == other->Position);
         }
 
-        /// <summary>
-        ///   check out if a bucket is within the same higher-level boundary as currentOffset
-        /// </summary>
-        private bool AssertBoundary(
-                     Entry* previous,
-                     Entry* current,
-                     int level
-        )
-        {
-            level = int.Min(level, MaxLevel);
-            uint nextLevelSize = _levelSizeArray[level] * 4;
-            ulong prevHead = previous->Position - ((ulong)m_Buffer),
-                currHead = current->Position - ((ulong)m_Buffer);
-            return currHead / nextLevelSize == prevHead / nextLevelSize;
-        }
-
-        /// <summary>
-        ///   Determine segment index based on size and originLevel
-        /// </summary>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int DetermineSegmentIndex(
-                           uint[] levelSizeArray,
-                           ulong size,
-                           int level
-        )
-        {
-            level = int.Min(level, MaxLevel);
-            return (int)((size - 1) / levelSizeArray[level]);
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static void InsertEntryAfter(
                             Entry* entry,
@@ -120,7 +93,7 @@ namespace StgSharp.HighPerformance.Memory
 
             nuint size = entry->Size;
             int originLevel = entry->Level;
-            int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
+            int segment = GetSegmentFromLevel(size, originLevel);
             do
             {
                 // previous direction merge
@@ -129,7 +102,7 @@ namespace StgSharp.HighPerformance.Memory
                     (p == _spareMemory) ||
                     (p->State != EntryState.Empty) ||
                     (p->Level != entry->Level) ||
-                    !AssertBoundary(p, entry, originLevel))
+                    !AssertAdjacentTo(p, entry))
                 {
                     break;
                 }
@@ -137,7 +110,7 @@ namespace StgSharp.HighPerformance.Memory
                 // Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
                 entry->Size += p->Size;
                 entry->Position = p->Position;
-                RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, p->Size, originLevel), (BucketNode*)p->Position);
+                RemoveFromBucket(originLevel, GetSegmentFromLevel(p->Size, originLevel), (BucketNode*)p->Position);
                 RemoveFromPositionChain(p);
             } while (true);
 
@@ -149,19 +122,18 @@ namespace StgSharp.HighPerformance.Memory
                     (n == _spareMemory) ||
                     (n->State != EntryState.Empty) ||
                     (n->Level != entry->Level) ||
-                    !AssertBoundary(n, entry, originLevel))
+                    !AssertAdjacentTo(n, entry))
                 {
                     break;
                 }
 
                 // Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
                 entry->Size += n->Size;
-                RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, n->Size, originLevel), (BucketNode*)n->Position);
+                RemoveFromBucket(originLevel, GetSegmentFromLevel(n->Size, originLevel), (BucketNode*)n->Position);
                 RemoveFromPositionChain(n);
             } while (true);
             originLevel = int.Min(9, originLevel);
-            entry->Level += (entry->Size >= _levelSizeArray[originLevel] * 4) ? 1 : 0;
+            entry->Level = GetLevelFromSize(entry->Size);
-            entry->Level &= 15;
 
             BucketNode* bucket = (BucketNode*)entry->Position;
             bucket->EntryRef = entry;
@@ -190,7 +162,7 @@ namespace StgSharp.HighPerformance.Memory
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static void SetEntryLevel(
+        private void SetEntryLevel(
                      Entry* entry,
                      nuint position,
                      nuint size

StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs

@@ -30,7 +30,6 @@ using System.Buffers;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
-using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
 
 namespace StgSharp.HighPerformance.Memory
 {
@@ -38,16 +37,17 @@ namespace StgSharp.HighPerformance.Memory
     {
 
         // Bucket system field - directly in main allocator class
-        private readonly BucketNode *[] _bucketHeads;
+        private readonly BucketNode*[] _bucketHeads;
 
         // _spareMemory is also the beginning position of allocator
-        private Entry *_spareMemory;
+        private Entry* _spareMemory;
         private readonly byte* m_Buffer;
 
         private readonly Action<nuint> _freeHandle;
         private bool disposedValue;
         private readonly int _align;
-        private readonly nuint _maxSize;
+        private readonly int _maxLevel;
+        private readonly nuint _maxAllocSize;
         private readonly nuint _size;
         private readonly SlabAllocator<Entry> _entries;
 
@@ -88,17 +88,9 @@ namespace StgSharp.HighPerformance.Memory
             _spareMemory->Position = (nuint)m_Buffer;
             _spareMemory->Size = byteSize;
 
-            int levelCountOrigin = _levelSizeArray.Length;
+            _maxLevel = GetLevelFromSize(maxBlockSize) + 1;
-            int levelCount = GetLevelFromSize(maxBlockSize) + 1;
+            _bucketHeads = new BucketNode*[(_maxLevel * 3) + 1];
-            Array.Resize(ref _levelSizeArray, levelCount);
+            _maxAllocSize = maxBlockSize;
-            _bucketHeads = new BucketNode*[(levelCount * 3) + 1];
-            uint size = _levelSizeArray[^1];
-            for (int i = levelCountOrigin; i < levelCount; i++)
-            {
-                size *= 4;
-                _levelSizeArray[i] = size;
-            }
-            _maxSize = maxBlockSize;
             _freeHandle = freeHandle ?? (static _ => { });
         }
 

StgSharp.Common/HighPerformance/Memory/L4.cs

@@ -43,6 +43,7 @@ namespace StgSharp.HighPerformance.Memory
         private const int PredictionCount = 2048;
         private const int PredictionLeastAhead = 1024;
 
+        // private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
         private readonly CacheLineHead* _head;
         private readonly MapArray* _mapCount;
         private readonly PredictionArray* _predictCount;

StgSharp.Mathematics/Internal/IntrinsicContext.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="IntrinsicContext"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -68,7 +68,9 @@ namespace StgSharp.Mathematics.Internal
 
         [FieldOffset(0)] public readonly ushort Size;
 
-        private MatrixElementType(int mask)
+        private MatrixElementType(
+                int mask
+        )
         {
             Unsafe.SkipInit(out this);
             Unsafe.SkipInit(out Size);
@@ -76,6 +78,24 @@ namespace StgSharp.Mathematics.Internal
             this.Mask = mask;
         }
 
+        [Obsolete("Do not use this constructor. Use pre-defined ones from static fields.", true)]
+        public MatrixElementType()
+        {
+            throw new NotSupportedException("Do not use this constructor. Use pre-defined ones from static fields.");
+        }
+
+        public override readonly bool Equals(
+                                      object? obj
+        )
+        {
+            return (obj is MatrixElementType type) && (Mask == type.Mask);
+        }
+
+        public override readonly int GetHashCode()
+        {
+            return Mask;
+        }
+
     }
 
     [StructLayout(LayoutKind.Sequential)]
@@ -105,14 +125,19 @@ namespace StgSharp.Mathematics.Internal
         private nint handle;              //f32_panel_mul
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public readonly void Call(MatrixIntrinsicHandle index, MatrixParallelTask* task)
+        public readonly void Call(
+                             MatrixIntrinsicHandle index,
+                             MatrixParallelTask* task
+        )
         {
             nint ptr = this[(int)index];
             ((delegate* unmanaged[Cdecl]<MatrixParallelTask*, void>)ptr)(task);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public readonly nint Get(MatrixIntrinsicHandle index)
+        public readonly nint Get(
+                             MatrixIntrinsicHandle index
+        )
         {
             return this[(int)index];
         }
@@ -143,15 +168,20 @@ namespace StgSharp.Mathematics.Internal
                              MatrixKernel* left,
                              MatrixKernel* right,
                              MatrixKernel* ans,
-                             ulong scalar)
+                             ulong scalar
+        )
         {
             nint ptr = this[(int)index];
-            ((delegate* unmanaged[Cdecl]<MatrixKernel*, MatrixKernel*, MatrixKernel*, ulong, void>)ptr)(left, right,
+            ((delegate* unmanaged[Cdecl]<MatrixKernel*, MatrixKernel*, MatrixKernel*, ulong, void>)ptr)(left,
-                                                                                                        ans, scalar);
+                                                                                                        right,
+                                                                                                        ans,
+                                                                                                        scalar);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public readonly nint Get(MatrixIntrinsicHandle index)
+        public readonly nint Get(
+                             MatrixIntrinsicHandle index
+        )
         {
             return this[(int)index];
         }

StgSharp.Mathematics/Numeric/Linear/MatrixCompute.cs

@@ -71,7 +71,7 @@ namespace StgSharp.Mathematics.Numeric
                 op(task);
                 return;
             }
-            MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferQueueLeftRightAns>(task);
+            MatrixParallelWrap taskWrap = MatrixParallel.ScheduleTask<BufferQueueLeftRightAns_NoNuma>(task);
             taskWrap.LaunchParallel(SleepMode.DeepSleep);
             taskWrap.Dispose();
         }

StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs

@@ -33,6 +33,7 @@ using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices.JavaScript;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -42,153 +43,85 @@ namespace StgSharp.Mathematics.Numeric
     internal unsafe interface IMatrixParallelQueue<TSelf> where TSelf : IMatrixParallelQueue<TSelf>
     {
 
-        static abstract TSelf Create(
+        bool InitNumaPredictor(
-                              MatrixParallelTask* baseTask,
+             out IL4Predict predictor
-                              L4 buffer,
-                              int numaCount
         );
 
-        void Return(
+        bool RequestTaskResource(
-             MatrixParallelTask* task,
-             in L4CacheLine left,
-             in L4CacheLine right,
-             in L4CacheLine ans
-        );
-
-        bool Steal(
              MatrixParallelTask* task,
              out L4CacheLine left,
              out L4CacheLine right,
              out L4CacheLine ans
         );
 
+        void ReturnTaskResource(
+             MatrixParallelTask* task,
+             in L4CacheLine left,
+             in L4CacheLine right,
+             in L4CacheLine ans
+        );
+
     }
 
-    internal sealed unsafe class BufferQueueLeftRightAns : IMatrixParallelQueue<BufferQueueLeftRightAns>
+    internal interface IBufferQueueLeftRightAns : IMatrixParallelQueue<IBufferQueueLeftRightAns> { }
+
+    internal sealed unsafe class BufferQueueLeftRightAns_NoNuma : IMatrixParallelQueue<IBufferQueueLeftRightAns>
     {
 
         private MatrixParallelTask* _baseTask;
-        private bool _needNuma;
+        private readonly nuint _byteSize;
-        private L4 _buffer;
+        private readonly nuint _count;
         private volatile nuint _cursor;
-        private readonly nuint Count;
 
-        private BufferQueueLeftRightAns() { }
+        private volatile nuint _mat1;
+        private volatile nuint _mat2;
+        private volatile nuint _mat3;
 
-        private BufferQueueLeftRightAns(
+        private BufferQueueLeftRightAns_NoNuma() { }
+
+        internal BufferQueueLeftRightAns_NoNuma(
                  MatrixParallelTask* baseTask
         )
         {
             _baseTask = baseTask;
-            Count = ((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY);
+            _count = ((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY);
+            _byteSize = ((nuint)_baseTask->Mat1) + (((nuint)_baseTask->CommonX) * ((nuint)_baseTask->CommonY) * _baseTask->ElementType.Size);
+
+
+            _mat1 = (nuint)_baseTask->Mat1;
+            _mat2 = (nuint)_baseTask->Mat2;
+            _mat3 = (nuint)_baseTask->Mat3;
         }
 
-        public static unsafe BufferQueueLeftRightAns Create(
+        public bool InitNumaPredictor(
-                                                     MatrixParallelTask* baseTask,
+                    out IL4Predict predictor
-                                                     L4 buffer,
-                                                     int numaCount
         )
         {
-            return new BufferQueueLeftRightAns(baseTask);
+            predictor = null;
+            return false;
         }
 
-        public unsafe void Return(
+        public unsafe bool RequestTaskResource(
-                           MatrixParallelTask* task,
-                           in L4CacheLine left,
-                           in L4CacheLine right,
-                           in L4CacheLine ans
-        )
-        {
-            if (left.Address != 0) {
-                left.Dispose();
-            }
-            if (right.Address != 0) {
-                right.Dispose();
-            }
-            if (ans.Address != 0) {
-                ans.Dispose();
-            }
-        }
-
-        public unsafe bool Steal(
                            MatrixParallelTask* task,
                            out L4CacheLine left,
                            out L4CacheLine right,
                            out L4CacheLine ans
         )
         {
-            throw new NotImplementedException();
+            left = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
+            right = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
+            ans = new L4CacheLine(0, ref Unsafe.AsRef<int>(null), ref Unsafe.AsRef<int>(null));
+            return true;
         }
 
-        private class Predictor(
+        public unsafe void ReturnTaskResource(
-                      BufferQueueLeftRightAns host
+                           MatrixParallelTask* task,
-        ) : IL4Predict
+                           in L4CacheLine left,
-        {
+                           in L4CacheLine right,
-
+                           in L4CacheLine ans
-            private readonly BufferQueueLeftRightAns _host;
-
-            public PredictResult Predict(
-                                 Span<CacheLineDescription> node,
-                                 out int count
         )
         {
-                nuint begin = _host._cursor;
+            return;
-                nuint mat1 = (nuint)_host._baseTask->Mat1;
-                nuint mat2 = (nuint)_host._baseTask->Mat2;
-                nuint mat3 = (nuint)_host._baseTask->Mat3;
-                nuint _cur = begin * _host._baseTask->ElementType.Size;
-                nuint mat1_cur = ((nuint)_host._baseTask->Mat1) + _cur;
-                nuint mat2_cur = ((nuint)_host._baseTask->Mat2) + _cur;
-                nuint mat3_cur = ((nuint)_host._baseTask->Mat3) + _cur;
-                nuint end = ((nuint)_host._baseTask->Mat1) + (((nuint)_host._baseTask->CommonX) * ((nuint)_host._baseTask->CommonY) * _host._baseTask->ElementType.Size);
-                nuint mat1_end = ((nuint)_host._baseTask->Mat1) + end;
-                nuint mat2_end = ((nuint)_host._baseTask->Mat2) + end;
-                nuint mat3_end = ((nuint)_host._baseTask->Mat3) + end;
-
-                nuint size = 1024 * 16 * _host._baseTask->ElementType.Size;
-                for (int i = 0; i < node.Length; i += 3)
-                {
-                    int cycle = i / 3;
-                    ref CacheLineDescription prediction = ref node[i];
-                    prediction.Address = mat1_cur + size;
-                }
-                count = 1;
-                return PredictResult.Success;
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void Prefetch(
-                        nuint origin,
-                        uint policy,
-                        Span<byte> cache
-            )
-            {
-                MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
-                if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
-                {
-                    if ((pEnum & (MatrixCachePolicy.Left | MatrixCachePolicy.Right)) != 0)
-                    {
-                        byte* bPtr = (byte*)origin;
-                        new Span<byte>(bPtr, cache.Length).CopyTo(cache);
-                    }
-                }
-            }
-
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void WriteBack(
-                        nuint origin,
-                        uint policy,
-                        ReadOnlySpan<byte> cache
-            )
-            {
-                MatrixCachePolicy pEnum = (MatrixCachePolicy)policy;
-                if ((pEnum & (MatrixCachePolicy.Ans | MatrixCachePolicy.Sequential)) != 0)
-                {
-                    byte* bPtr = (byte*)origin;
-                    cache.CopyTo(new Span<byte>(bPtr, cache.Length));
-                }
-            }
-
         }
 
     }

StgSharp.Mathematics/Numeric/Runtime/MatrixParallelTask.cs

@@ -52,20 +52,20 @@ namespace StgSharp.Mathematics.Numeric.Runtime
         [FieldOffset(8)] public MatrixKernel* Mat2;
         [FieldOffset(16)] public MatrixKernel* Mat3;
         /// <summary>
-        ///   Being offset of first kernel in a L4 cache line.
-        /// </summary>
-        [FieldOffset(24)] public nuint CacheIndex;
-        /// <summary>
         ///   Count of kernels to be calculated in a L4 cache line.
         /// </summary>
         [FieldOffset(28)] public int CacheLength;
         [FieldOffset(32)] public int CommonK;
         [FieldOffset(36)] public int CommonX;
         [FieldOffset(40)] public int CommonY;
-        [FieldOffset(44)] public int SpanLength;
+        [FieldOffset(44)] public int VectorLength;
         [FieldOffset(48)] public MatrixElementType ElementType;
         [FieldOffset(52)] public MatrixIntrinsicHandle ComputeHandle;
-        [FieldOffset(56)] public nint Span;
+        /// <summary>
+        ///   Beginning offset of first kernel in a L4 cache line.
+        /// </summary>
+        [FieldOffset(24)] public nuint Offset;
+        [FieldOffset(56)] public nuint Vector;
         [FieldOffset(64)] public ScalarPacket Scalar;
 
         public MatrixParallelTask()

StgSharp.Mathematics/Numeric/Runtime/QueueFactory.cs

@@ -0,0 +1,42 @@
+//-----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+// file="QueueFactory"
+// Project: StgSharp
+// AuthorGroup: Nitload
+// Copyright (c) Nitload. All rights reserved.
+//     
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+//     
+// -----------------------------------------------------------------------
+// -----------------------------------------------------------------------
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StgSharp.Mathematics.Numeric
+{
+    internal interface IMatrixQueueFactory
+    {
+
+        IMatrixParallelQueue<IMatrixParallelQueue> CreateSequential();
+
+    }
+}

StgSharp.Native/compile_commands.json

@@ -41,6 +41,12 @@
   "file": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-src/mk.fma.x86.c",
   "output": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu/CMakeFiles/StgSharp.Native.dir/sn-src/mk.fma.x86.c.obj"
 },
+{
+  "directory": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu",
+  "command": "C:\\PROGRA~1\\LLVM\\bin\\clang.exe -DSN_ARCH_TARGET=SN_ARCH_X86_64 -DSN_EXTERN_TARGET -DSN_OS_TARGET=SN_OS_WINDOWS -DStgSharp_Native_EXPORTS -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-head -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-template -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/glfw -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/stbi -O0 -g -Xclang -gcodeview -std=gnu11 -D_DEBUG -D_DLL -D_MT -Xclang --dependent-lib=msvcrtd -mavx -mavx2 -mfma -mavx512f -mavx512vl -o CMakeFiles\\StgSharp.Native.dir\\sn-src\\mk.p_scan.x86.c.obj -c D:\\NitloadIndustry\\研究项目\\1-Flagship\\TH2020\\ProjectFile\\内核源代码\\StgSharp.Native\\sn-src\\mk.p_scan.x86.c",
+  "file": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-src/mk.p_scan.x86.c",
+  "output": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu/CMakeFiles/StgSharp.Native.dir/sn-src/mk.p_scan.x86.c.obj"
+},
 {
   "directory": "D:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/build/cmake-clang-gnu",
   "command": "C:\\PROGRA~1\\LLVM\\bin\\clang.exe -DSN_ARCH_TARGET=SN_ARCH_X86_64 -DSN_EXTERN_TARGET -DSN_OS_TARGET=SN_OS_WINDOWS -DStgSharp_Native_EXPORTS -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-head -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/sn-template -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/glfw -ID:/NitloadIndustry/研究项目/1-Flagship/TH2020/ProjectFile/内核源代码/StgSharp.Native/lib/stbi -O0 -g -Xclang -gcodeview -std=gnu11 -D_DEBUG -D_DLL -D_MT -Xclang --dependent-lib=msvcrtd -mavx -mavx2 -mfma -mavx512f -mavx512vl -o CMakeFiles\\StgSharp.Native.dir\\sn-src\\s.factorial.x86.c.obj -c D:\\NitloadIndustry\\研究项目\\1-Flagship\\TH2020\\ProjectFile\\内核源代码\\StgSharp.Native\\sn-src\\s.factorial.x86.c",