tlsf briefly ok, debugging

StgSharp.Common/HighPerformance/Memory/L4.Evict.cs

@@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory
             }
 
             // origin :CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
-            CacheLineHead* entry = (CacheLineHead*)handle.Pointer;
+            CacheLineHead* entry = (CacheLineHead*)handle.Address;
             index = entry->Id;
             return true;
         }

StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs

@@ -50,6 +50,7 @@ namespace StgSharp.HighPerformance.Memory
         private readonly Action<nuint> _freeHandle;
         private bool disposedValue;
         private readonly int _align;
+        private readonly int _maxLevel;
 
         private int _metadataEnd;
         private readonly Ptr64 _basePtr;
@@ -77,15 +78,17 @@ namespace StgSharp.HighPerformance.Memory
             _basePtr = (Ptr64)(ulong)externalPointer;
             _align = align;
             _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
-            _bucket = new int[GetLevel(_maxAllocSize) + 1];
+            _maxLevel = int.MaxValue;
+            _maxLevel = GetLevel(_maxAllocSize);
+            _bucket = new int[GetLevel(_maxAllocSize) + 2];
             _spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
             _freeHandle = freeHandle ?? (static_ => { });
             _metadataEnd = 0;
 
             // init spare memory node
 
-            _spareMemory->Position = 0UL;
+            _spareMemory->Position = _basePtr;
-            _spareMemory->Size = _totalSize - Metadata.SizeOf;
+            _spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
             _spareMemory->NextLevel = 0;
             _spareMemory->NextNear = 0;
             _spareMemory->PreviousLevel = 0;
@@ -120,6 +123,242 @@ namespace StgSharp.HighPerformance.Memory
             : this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
                    ptr => NativeMemory.AlignedFree((void*)ptr)) { }
 
+        public void Dispose()
+        {
+            Dispose(disposing:true);
+            GC.SuppressFinalize(this);
+        }
+
+        protected virtual void Dispose(
+                               bool disposing
+        )
+        {
+            if (!disposedValue)
+            {
+                if (disposing) {
+                    _freeHandle((nuint)_basePtr);
+                }
+
+                disposedValue = true;
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetLevel(
+                    ulong size
+        )
+        {
+            int origin = 57 - BitOperations.LeadingZeroCount(size);
+            return int.Min(origin, _maxLevel);
+        }
+
+        /// <summary>
+        ///   Put a metadata node back to free list and waiting for next allocation.
+        /// </summary>
+        /// <param name="node">
+        ///
+        /// </param>
+        private void RecycleNode(
+                     Metadata* node
+        )
+        {
+            int nextIdx = _spareMemory->NextLevel;
+            int nodeIdx = (int)(_spareMemory - node);
+            int level = GetLevel(node->Size);
+            Metadata* next = &_spareMemory[-nextIdx];
+
+            int nextLevelIdx = node->NextLevel;
+            int prevLevelIdx = node->PreviousLevel;
+            if (nextIdx == 0)
+            {
+                _spareMemory->NextLevel = nodeIdx;
+                node->NextLevel = 0;
+                return;
+            } else
+            {
+                // add to free list
+                next->PreviousLevel = nodeIdx;
+                _spareMemory->NextLevel = nodeIdx;
+                node->NextLevel = nextIdx;
+                node->PreviousLevel = 0;
+            }
+
+            // remove from position chain
+            int prevNearIdx = node->PreviousNear;
+            int nextNearIdx = node->NextNear;
+            Metadata* prevNear = &_spareMemory[-prevNearIdx];
+            Metadata* nextNear = &_spareMemory[-nextNearIdx];
+            prevNear->NextNear = nextNearIdx;
+            nextNear->PreviousNear = prevNearIdx;
+            node->PreviousNear = 0;
+            node->NextNear = 0;
+
+            node->Position = 0;
+            node->SizeForSpare = 0;
+
+            // remove from level bucket
+            if (_bucket[level] == nodeIdx)
+            {
+                _bucket[level] = nextLevelIdx;
+                if (nextLevelIdx != 0)
+                {
+                    Metadata* nextLevel = &_spareMemory[-nextLevelIdx];
+                    nextLevel->PreviousLevel = 0;
+                }
+            } else
+            {
+                if (nextLevelIdx != 0)
+                {
+                    Metadata* nextLevel = &_spareMemory[-nextLevelIdx];
+                    nextLevel->PreviousLevel = prevLevelIdx;
+                }
+                if (prevLevelIdx != 0)
+                {
+                    Metadata* prevLevel = &_spareMemory[-prevLevelIdx];
+                    prevLevel->NextLevel = nextLevelIdx;
+                }
+                node->PreviousLevel = 0;
+                node->NextLevel = 0;
+            }
+        }
+
+        private bool TryAllocNewChunk(
+                     ulong size,
+                     out Metadata* meta
+        )
+        {
+            int idx = _spareMemory->NextLevel;
+            if (idx != 0)
+            {
+                meta = &_spareMemory[-idx];
+                _spareMemory->NextLevel = meta->NextLevel;
+            } else
+            {
+                _metadataEnd++;
+                meta = &_spareMemory[-_metadataEnd];
+
+                // TODO: correctly set position and size for new chunk
+            }
+            ulong spareSize = _spareMemory->Size;
+            if (spareSize < Metadata.SizeOf + size)
+            {
+                meta = null;
+                return false;
+            }
+            spareSize -= Metadata.SizeOf + size;
+            _spareMemory->Size = spareSize;
+            _spareMemory->Position += size;
+            meta->Size = size;
+
+            // insert the new chunk to position chain
+            int nearIdx = _spareMemory->PreviousNear;
+            Metadata* near = &_spareMemory[-nearIdx];
+            near->NextNear = _metadataEnd;
+            meta->PreviousNear = nearIdx;
+            _spareMemory->PreviousNear = _metadataEnd;
+            meta->NextNear = 0;
+            return true;
+        }
+
+        private bool TryGetNewMetadata(
+                     out Metadata* meta
+        )
+        {
+            ulong spareSize = _spareMemory->Size;
+            if (spareSize < Metadata.SizeOf)
+            {
+                meta = null;
+                return false;
+            }
+            _metadataEnd++;
+            meta = &_spareMemory[-_metadataEnd];
+            spareSize -= Metadata.SizeOf;
+            _spareMemory->Size = spareSize;
+            return true;
+        }
+
+        [StructLayout(LayoutKind.Explicit)]
+        public readonly struct Handle
+        {
+
+            [FieldOffset(0)] internal readonly Metadata* _meta;
+            [FieldOffset(8)] public readonly void* Address;
+            [FieldOffset(16)] public readonly Ptr64 Size;
+
+            internal Handle(
+                     Metadata* meta,
+                     Ptr64 baseAddress
+            )
+            {
+                _meta = meta;
+                Address = (void*)(ulong)meta->Position;
+                Size = meta->Size;
+            }
+
+            [Obsolete("You should not create instance from default. Alloc from allocator please")]
+            public Handle() { }
+
+            public ref byte this[
+                            int offset
+            ] => ref ((byte*)Address)[offset];
+
+            public Span<byte> AsSpan(
+                              int range = -1
+            )
+            {
+                return new Span<byte>(Address, range < 0 ? (int)(ulong)Size : range);
+            }
+
+        }
+
+        internal enum AllocationState : byte
+        {
+
+            SpareOrCollect = 0,
+            Free = 1,
+            Alloc = 2,
+
+        }
+
+        [StructLayout(LayoutKind.Explicit, Size = 32)]
+        internal struct Metadata
+        {
+
+            [FieldOffset(24)] private ulong PositionMask;
+            [FieldOffset(16)] private ulong SizeMask;
+
+            [FieldOffset(30)] internal AllocationState State;
+            [FieldOffset(31)] internal sbyte Level;
+
+            [FieldOffset(0)] public int NextLevel;
+            [FieldOffset(4)] public int NextNear;
+            [FieldOffset(8)] public int PreviousLevel;
+            [FieldOffset(12)] public int PreviousNear;
+
+            public Ptr64 Position
+            {
+                readonly get => PositionMask;
+                set => PositionMask = value;
+            }
+
+            public Ptr64 Size
+            {
+                readonly get => (Ptr64)(SizeMask & Mask48Bits);
+                set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
+            }
+
+            public Ptr64 SizeForSpare
+            {
+                readonly get => SizeMask;
+                set => SizeMask = value;
+            }
+
+            public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
+
+        }
+
+        #region alloc and free
+
         public Handle Alloc(
                       nuint size
         )
@@ -176,15 +415,37 @@ namespace StgSharp.HighPerformance.Memory
                 entry->NextNear = newEntryIdx;
                 newEntry->PreviousNear = entryIdx;
             }
+            entry->State = AllocationState.Alloc;
             return new Handle(entry, _basePtr);
         }
 
-        public void Collect() { }
+        public void Collect()
-
-        public void Dispose()
         {
-            Dispose(disposing:true);
+            int currentIdx = _spareMemory->NextNear;
-            GC.SuppressFinalize(this);
+            do
+            {
+                if (currentIdx == 0)
+                {
+                    break;
+                }
+                Metadata* current = &_spareMemory[-currentIdx];
+                if (current->State == AllocationState.Free)
+                {
+                    Ptr64 size = 0;
+                    int nextIdx = current->NextNear;
+                    Metadata* next = &_spareMemory[-nextIdx];
+                    while (nextIdx != 0 && next->State != AllocationState.Free)
+                    {
+                        size += next->Size;
+                        RecycleNode(next);
+                        nextIdx = next->NextNear;
+                    }
+                    if (size > 0) {
+                        current->Size += size;
+                    }
+                }
+                currentIdx = current->NextNear;
+            } while (true);
         }
 
         public void Free(
@@ -193,14 +454,55 @@ namespace StgSharp.HighPerformance.Memory
         )
         {
             Metadata* head = handle._meta;
+            int headIdx = (int)(_spareMemory - head);
+            Ptr64 size = handle.Size;
             for (int i = 0; i < collectScanWindow; i++)
             {
-                // forward scan
+                // forward scan, previous direction
+                int forwardIdx = head->PreviousNear;
+                if (forwardIdx == 0)
+                {
+                    break;
+                }
+                Metadata* prev = &_spareMemory[-forwardIdx];
+                if (prev->State != AllocationState.Free)
+                {
+                    break;
+                }
+                size += prev->Size;
+                head->Position = prev->Position;
+                RecycleNode(prev);
             }
             for (int i = 0; i < collectScanWindow; i++)
             {
-                // backward scan
+                // backward scan, next direction
+                int nextIdx = head->NextNear;
+                if (nextIdx == 0)
+                {
+                    break;
                 }
+                Metadata* prev = &_spareMemory[-nextIdx];
+                if (prev->State != AllocationState.Free)
+                {
+                    break;
+                }
+                size += prev->Size;
+                RecycleNode(prev);
+            }
+
+            // recycle to level;
+            head->Size = size;
+            head->State = AllocationState.Free;
+            int level = GetLevel(head->Size);
+            int bucketHeadIdx = _bucket[level];
+            if (bucketHeadIdx != 0)
+            {
+                Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
+                bucketHead->PreviousLevel = headIdx;
+                head->NextLevel = bucketHeadIdx;
+            }
+            _bucket[level] = headIdx;
+            return;
         }
 
         public bool TryAlloc(
@@ -208,162 +510,66 @@ namespace StgSharp.HighPerformance.Memory
                     out Handle h
         )
         {
-            throw new NotImplementedException();
+            ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
-        }
+            if (allocSize > _maxAllocSize)
-
-        protected virtual void Dispose(
-                               bool disposing
-        )
             {
-            if (!disposedValue)
+                h = default;
+                return false;
+            }
+            int level = GetLevel(allocSize);
+            Metadata* entry = null;
+            for (int i = level; i < _bucket.Length; i++)
             {
-                if (disposing) {
+                int idx = _bucket[i];
-                    _freeHandle((nuint)_basePtr);
-                }
-
-                disposedValue = true;
-            }
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int GetLevel(
-                           ulong size
-        )
-        {
-            return 57 - BitOperations.LeadingZeroCount(size);
-        }
-
-        private bool TryAllocNewChunk(
-                     ulong size,
-                     out Metadata* meta
-        )
-        {
-            int idx = _spareMemory->NextLevel;
                 if (idx != 0)
                 {
-                meta = &_spareMemory[-idx];
+                    entry = &_spareMemory[-idx];
-                _spareMemory->NextLevel = meta->NextLevel;
+                    _bucket[i] = entry->NextLevel;
-                return true;
+                    break;
                 }
-            ulong spareSize = _spareMemory->Size;
+            }
-            if (spareSize < Metadata.SizeOf + size)
+            if (entry is null && !TryAllocNewChunk(allocSize, out entry))
             {
-                meta = null;
+                h = default;
                 return false;
             }
-            _metadataEnd++;
+            int entryIdx = (int)(_spareMemory - entry);
-            meta = &_spareMemory[-_metadataEnd];
+            ulong entrySize = entry->Size;
-            spareSize -= Metadata.SizeOf + size;
+            ulong remain = entrySize - allocSize;
-            _spareMemory->Size = spareSize;
-            _spareMemory->Position += size;
-            meta->Size = size;
 
-            // insert the new chunk to position chain
+            // slice tail
-            int nearIdx = _spareMemory->PreviousNear;
+            if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
-            Metadata* near = &_spareMemory[-nearIdx];
+            {
-            near->NextNear = _metadataEnd;
+                int newEntryIdx = (int)(_spareMemory - newEntry);
-            meta->PreviousNear = nearIdx;
+                newEntry->Position = entry->Position + allocSize;
-            _spareMemory->PreviousNear = _metadataEnd;
+                newEntry->Size = remain;
-            meta->NextNear = 0;
+                int newLevel = GetLevel(remain);
+                int bucketHeadIdx = _bucket[newLevel];
+                if (bucketHeadIdx != 0)
+                {
+                    // if there is already an entry in the bucket, link it to the new entry
+                    Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
+                    newEntry->NextLevel = bucketHeadIdx;
+                    bucketHead->PreviousLevel = newEntryIdx;
+                } else
+                {
+                    newEntry->NextLevel = 0;
+                }
+                newEntry->PreviousLevel = 0;
+                _bucket[newLevel] = newEntryIdx;
+
+                // push to position chain
+                int afterIdx = entry->NextNear;
+                Metadata* after = &_spareMemory[-afterIdx];
+                newEntry->NextNear = afterIdx;
+                after->PreviousNear = newEntryIdx;
+                entry->NextNear = newEntryIdx;
+                newEntry->PreviousNear = entryIdx;
+            }
+            h = new Handle(entry, _basePtr);
             return true;
         }
 
-        private bool TryGetNewMetadata(
+        #endregion
-                     out Metadata* meta
-        )
-        {
-            ulong spareSize = _spareMemory->Size;
-            if (spareSize < Metadata.SizeOf)
-            {
-                meta = null;
-                return false;
-            }
-            _metadataEnd++;
-            meta = &_spareMemory[-_metadataEnd];
-            spareSize -= Metadata.SizeOf;
-            _spareMemory->Size = spareSize;
-            return true;
-        }
-
-        [StructLayout(LayoutKind.Explicit)]
-        public readonly struct Handle
-        {
-
-            [FieldOffset(0)]internal readonly Metadata* _meta;
-            [FieldOffset(8)]public readonly void* Pointer;
-            [FieldOffset(16)]public readonly Ptr64 Size;
-
-            internal Handle(
-                     Metadata* meta,
-                     Ptr64 baseAddress
-            )
-            {
-                _meta = meta;
-                Pointer = (void*)(ulong)(meta->Position + baseAddress);
-                Size = meta->Size;
-            }
-
-            [Obsolete("You should not create instance from default. Alloc from allocator please")]
-            public Handle() { }
-
-            public ref byte this[
-                            int offset
-            ] => ref ((byte*)Pointer)[offset];
-
-            public Span<byte> AsSpan(
-                              int range = -1
-            )
-            {
-                return new Span<byte>(Pointer, range < 0 ? (int)(ulong)Size : range);
-            }
-
-        }
-
-        internal enum AllocationState : short
-        {
-
-            Free = 0,
-            Used = 1,
-            Split = 2,
-            Merged = 3
-
-        }
-
-        [StructLayout(LayoutKind.Explicit, Size = 32)]
-        internal struct Metadata
-        {
-
-            [FieldOffset(24)] private ulong PositionMask;
-            [FieldOffset(16)] private ulong SizeMask;
-
-            [FieldOffset(30)] internal AllocationState State;
-
-            [FieldOffset(0)] public int NextLevel;
-            [FieldOffset(4)] public int NextNear;
-            [FieldOffset(8)] public int PreviousLevel;
-            [FieldOffset(12)] public int PreviousNear;
-
-            public Ptr64 Position
-            {
-                readonly get => PositionMask;
-                set => PositionMask = value;
-            }
-
-            public Ptr64 Size
-            {
-                readonly get => (Ptr64)(SizeMask & Mask48Bits);
-                set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
-            }
-
-            public Ptr64 SizeForSpare
-            {
-                readonly get => SizeMask;
-                set => SizeMask = value;
-            }
-
-            public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
-
-        }
-
     }
 }

StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs

@@ -62,7 +62,7 @@ namespace StgSharp.Mathematics.Numeric
 
         internal readonly Tlsf _allocator = allocator;
 
-        protected internal override unsafe T* BufferPointer => (T*)_handle.Pointer;
+        protected internal override unsafe T* BufferPointer => (T*)_handle.Address;
 
         public override void Dispose()
         {

StgSharpDebug/Program.cs

@@ -53,9 +53,9 @@ namespace StgSharp
                      .InitializeModule();
 
                 // TestRegexAnalyse.Run();
-                TestHlsf();
+                // TestHlsf();
 
-                // HlsfStressTest.Run();
+                TlsfStressTest.Run();
 
                 // TestSwiss();
                 // TestL4Prefetch();

StgSharpDebug/hlsf-test.cs

@@ -84,11 +84,18 @@ namespace StgSharp
                 {
                     List<Tlsf.Handle> batch = new(maxCount);
                     for (int i = 0; i < maxCount; i++)
+                    {
+                        try
                         {
                             uint sz = sizes[rnd.Next(sizes.Length)];
                             batch.Add(hlsfAllocator.Alloc(sz));
-
+                        }
-                        // Console.WriteLine($"alloced:{i}");
+                        catch (Exception e)
+                        {
+                            Console.WriteLine($"alloced:{round},{i}");
+                            Console.WriteLine(e);
+                            throw;
+                        }
                     }
                     for (int i = 5; i < maxCount; i += 10)
                     {
@@ -98,10 +105,17 @@ namespace StgSharp
                         batch[index] = origin;
                     }
                     for (int i = batch.Count - 1; i >= 0; i--)
+                    {
+                        try
                         {
                             hlsfAllocator.Free(batch[i]);
-
+                        }
-                        // Console.WriteLine($"freed:{i}");
+                        catch (Exception e)
+                        {
+                            Console.WriteLine($"freed:{round},{i}");
+                            Console.WriteLine(e);
+                            throw;
+                        }
                     }
                     if (round % 100 == 0)
                     {