From bf9879f59fcad5e547ca7ca4c58cf2cd2aed65d2 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Tue, 12 May 2026 18:25:48 +0800 Subject: [PATCH] tlsf briefly ok, debugging --- .../HighPerformance/Memory/L4.Evict.cs | 2 +- .../Memory/TwoLayerSegregatedFitAllocator.cs | 520 ++++++++++++------ .../Numeric/Linear/MatrixStorage.cs | 2 +- StgSharpDebug/Program.cs | 4 +- StgSharpDebug/hlsf-test.cs | 28 +- 5 files changed, 388 insertions(+), 168 deletions(-) diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index e895415b..e900c2e1 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory } // origin :CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle; - CacheLineHead* entry = (CacheLineHead*)handle.Pointer; + CacheLineHead* entry = (CacheLineHead*)handle.Address; index = entry->Id; return true; } diff --git a/StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs b/StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs index e31a2480..772d5f42 100644 --- a/StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs +++ b/StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs @@ -50,6 +50,7 @@ namespace StgSharp.HighPerformance.Memory private readonly Action _freeHandle; private bool disposedValue; private readonly int _align; + private readonly int _maxLevel; private int _metadataEnd; private readonly Ptr64 _basePtr; @@ -77,15 +78,17 @@ namespace StgSharp.HighPerformance.Memory _basePtr = (Ptr64)(ulong)externalPointer; _align = align; _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits))); - _bucket = new int[GetLevel(_maxAllocSize) + 1]; + _maxLevel = int.MaxValue; + _maxLevel = GetLevel(_maxAllocSize); + _bucket = new int[GetLevel(_maxAllocSize) + 2]; _spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1]; _freeHandle = freeHandle ?? (static_ => { }); _metadataEnd = 0; // init spare memory node - _spareMemory->Position = 0UL; - _spareMemory->Size = _totalSize - Metadata.SizeOf; + _spareMemory->Position = _basePtr; + _spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf; _spareMemory->NextLevel = 0; _spareMemory->NextNear = 0; _spareMemory->PreviousLevel = 0; @@ -120,6 +123,242 @@ namespace StgSharp.HighPerformance.Memory : this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { } + public void Dispose() + { + Dispose(disposing:true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose( + bool disposing + ) + { + if (!disposedValue) + { + if (disposing) { + _freeHandle((nuint)_basePtr); + } + + disposedValue = true; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetLevel( + ulong size + ) + { + int origin = 57 - BitOperations.LeadingZeroCount(size); + return int.Min(origin, _maxLevel); + } + + /// + /// Put a metadata node back to free list and waiting for next allocation. + /// + /// + /// + /// + private void RecycleNode( + Metadata* node + ) + { + int nextIdx = _spareMemory->NextLevel; + int nodeIdx = (int)(_spareMemory - node); + int level = GetLevel(node->Size); + Metadata* next = &_spareMemory[-nextIdx]; + + int nextLevelIdx = node->NextLevel; + int prevLevelIdx = node->PreviousLevel; + if (nextIdx == 0) + { + _spareMemory->NextLevel = nodeIdx; + node->NextLevel = 0; + return; + } else + { + // add to free list + next->PreviousLevel = nodeIdx; + _spareMemory->NextLevel = nodeIdx; + node->NextLevel = nextIdx; + node->PreviousLevel = 0; + } + + // remove from position chain + int prevNearIdx = node->PreviousNear; + int nextNearIdx = node->NextNear; + Metadata* prevNear = &_spareMemory[-prevNearIdx]; + Metadata* nextNear = &_spareMemory[-nextNearIdx]; + prevNear->NextNear = nextNearIdx; + nextNear->PreviousNear = prevNearIdx; + node->PreviousNear = 0; + node->NextNear = 0; + + node->Position = 0; + node->SizeForSpare = 0; + + // remove from level bucket + if (_bucket[level] == nodeIdx) + { + _bucket[level] = nextLevelIdx; + if (nextLevelIdx != 0) + { + Metadata* nextLevel = &_spareMemory[-nextLevelIdx]; + nextLevel->PreviousLevel = 0; + } + } else + { + if (nextLevelIdx != 0) + { + Metadata* nextLevel = &_spareMemory[-nextLevelIdx]; + nextLevel->PreviousLevel = prevLevelIdx; + } + if (prevLevelIdx != 0) + { + Metadata* prevLevel = &_spareMemory[-prevLevelIdx]; + prevLevel->NextLevel = nextLevelIdx; + } + node->PreviousLevel = 0; + node->NextLevel = 0; + } + } + + private bool TryAllocNewChunk( + ulong size, + out Metadata* meta + ) + { + int idx = _spareMemory->NextLevel; + if (idx != 0) + { + meta = &_spareMemory[-idx]; + _spareMemory->NextLevel = meta->NextLevel; + } else + { + _metadataEnd++; + meta = &_spareMemory[-_metadataEnd]; + + // TODO: correctly set position and size for new chunk + } + ulong spareSize = _spareMemory->Size; + if (spareSize < Metadata.SizeOf + size) + { + meta = null; + return false; + } + spareSize -= Metadata.SizeOf + size; + _spareMemory->Size = spareSize; + _spareMemory->Position += size; + meta->Size = size; + + // insert the new chunk to position chain + int nearIdx = _spareMemory->PreviousNear; + Metadata* near = &_spareMemory[-nearIdx]; + near->NextNear = _metadataEnd; + meta->PreviousNear = nearIdx; + _spareMemory->PreviousNear = _metadataEnd; + meta->NextNear = 0; + return true; + } + + private bool TryGetNewMetadata( + out Metadata* meta + ) + { + ulong spareSize = _spareMemory->Size; + if (spareSize < Metadata.SizeOf) + { + meta = null; + return false; + } + _metadataEnd++; + meta = &_spareMemory[-_metadataEnd]; + spareSize -= Metadata.SizeOf; + _spareMemory->Size = spareSize; + return true; + } + + [StructLayout(LayoutKind.Explicit)] + public readonly struct Handle + { + + [FieldOffset(0)] internal readonly Metadata* _meta; + [FieldOffset(8)] public readonly void* Address; + [FieldOffset(16)] public readonly Ptr64 Size; + + internal Handle( + Metadata* meta, + Ptr64 baseAddress + ) + { + _meta = meta; + Address = (void*)(ulong)meta->Position; + Size = meta->Size; + } + + [Obsolete("You should not create instance from default. Alloc from allocator please")] + public Handle() { } + + public ref byte this[ + int offset + ] => ref ((byte*)Address)[offset]; + + public Span AsSpan( + int range = -1 + ) + { + return new Span(Address, range < 0 ? (int)(ulong)Size : range); + } + + } + + internal enum AllocationState : byte + { + + SpareOrCollect = 0, + Free = 1, + Alloc = 2, + + } + + [StructLayout(LayoutKind.Explicit, Size = 32)] + internal struct Metadata + { + + [FieldOffset(24)] private ulong PositionMask; + [FieldOffset(16)] private ulong SizeMask; + + [FieldOffset(30)] internal AllocationState State; + [FieldOffset(31)] internal sbyte Level; + + [FieldOffset(0)] public int NextLevel; + [FieldOffset(4)] public int NextNear; + [FieldOffset(8)] public int PreviousLevel; + [FieldOffset(12)] public int PreviousNear; + + public Ptr64 Position + { + readonly get => PositionMask; + set => PositionMask = value; + } + + public Ptr64 Size + { + readonly get => (Ptr64)(SizeMask & Mask48Bits); + set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits); + } + + public Ptr64 SizeForSpare + { + readonly get => SizeMask; + set => SizeMask = value; + } + + public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); + + } + + #region alloc and free + public Handle Alloc( nuint size ) @@ -176,15 +415,37 @@ namespace StgSharp.HighPerformance.Memory entry->NextNear = newEntryIdx; newEntry->PreviousNear = entryIdx; } + entry->State = AllocationState.Alloc; return new Handle(entry, _basePtr); } - public void Collect() { } - - public void Dispose() + public void Collect() { - Dispose(disposing:true); - GC.SuppressFinalize(this); + int currentIdx = _spareMemory->NextNear; + do + { + if (currentIdx == 0) + { + break; + } + Metadata* current = &_spareMemory[-currentIdx]; + if (current->State == AllocationState.Free) + { + Ptr64 size = 0; + int nextIdx = current->NextNear; + Metadata* next = &_spareMemory[-nextIdx]; + while (nextIdx != 0 && next->State != AllocationState.Free) + { + size += next->Size; + RecycleNode(next); + nextIdx = next->NextNear; + } + if (size > 0) { + current->Size += size; + } + } + currentIdx = current->NextNear; + } while (true); } public void Free( @@ -193,14 +454,55 @@ namespace StgSharp.HighPerformance.Memory ) { Metadata* head = handle._meta; + int headIdx = (int)(_spareMemory - head); + Ptr64 size = handle.Size; for (int i = 0; i < collectScanWindow; i++) { - // forward scan + // forward scan, previous direction + int forwardIdx = head->PreviousNear; + if (forwardIdx == 0) + { + break; + } + Metadata* prev = &_spareMemory[-forwardIdx]; + if (prev->State != AllocationState.Free) + { + break; + } + size += prev->Size; + head->Position = prev->Position; + RecycleNode(prev); } for (int i = 0; i < collectScanWindow; i++) { - // backward scan + // backward scan, next direction + int nextIdx = head->NextNear; + if (nextIdx == 0) + { + break; + } + Metadata* prev = &_spareMemory[-nextIdx]; + if (prev->State != AllocationState.Free) + { + break; + } + size += prev->Size; + RecycleNode(prev); } + + // recycle to level; + head->Size = size; + head->State = AllocationState.Free; + int level = GetLevel(head->Size); + int bucketHeadIdx = _bucket[level]; + if (bucketHeadIdx != 0) + { + Metadata* bucketHead = &_spareMemory[-bucketHeadIdx]; + bucketHead->PreviousLevel = headIdx; + head->NextLevel = bucketHeadIdx; + } + _bucket[level] = headIdx; + return; } public bool TryAlloc( @@ -208,162 +510,66 @@ namespace StgSharp.HighPerformance.Memory out Handle h ) { - throw new NotImplementedException(); - } - - protected virtual void Dispose( - bool disposing - ) - { - if (!disposedValue) + ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63)); + if (allocSize > _maxAllocSize) { - if (disposing) { - _freeHandle((nuint)_basePtr); + h = default; + return false; + } + int level = GetLevel(allocSize); + Metadata* entry = null; + for (int i = level; i < _bucket.Length; i++) + { + int idx = _bucket[i]; + if (idx != 0) + { + entry = &_spareMemory[-idx]; + _bucket[i] = entry->NextLevel; + break; } - - disposedValue = true; } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetLevel( - ulong size - ) - { - return 57 - BitOperations.LeadingZeroCount(size); - } - - private bool TryAllocNewChunk( - ulong size, - out Metadata* meta - ) - { - int idx = _spareMemory->NextLevel; - if (idx != 0) + if (entry is null && !TryAllocNewChunk(allocSize, out entry)) { - meta = &_spareMemory[-idx]; - _spareMemory->NextLevel = meta->NextLevel; - return true; - } - ulong spareSize = _spareMemory->Size; - if (spareSize < Metadata.SizeOf + size) - { - meta = null; + h = default; return false; } - _metadataEnd++; - meta = &_spareMemory[-_metadataEnd]; - spareSize -= Metadata.SizeOf + size; - _spareMemory->Size = spareSize; - _spareMemory->Position += size; - meta->Size = size; + int entryIdx = (int)(_spareMemory - entry); + ulong entrySize = entry->Size; + ulong remain = entrySize - allocSize; - // insert the new chunk to position chain - int nearIdx = _spareMemory->PreviousNear; - Metadata* near = &_spareMemory[-nearIdx]; - near->NextNear = _metadataEnd; - meta->PreviousNear = nearIdx; - _spareMemory->PreviousNear = _metadataEnd; - meta->NextNear = 0; + // slice tail + if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry)) + { + int newEntryIdx = (int)(_spareMemory - newEntry); + newEntry->Position = entry->Position + allocSize; + newEntry->Size = remain; + int newLevel = GetLevel(remain); + int bucketHeadIdx = _bucket[newLevel]; + if (bucketHeadIdx != 0) + { + // if there is already an entry in the bucket, link it to the new entry + Metadata* bucketHead = &_spareMemory[-bucketHeadIdx]; + newEntry->NextLevel = bucketHeadIdx; + bucketHead->PreviousLevel = newEntryIdx; + } else + { + newEntry->NextLevel = 0; + } + newEntry->PreviousLevel = 0; + _bucket[newLevel] = newEntryIdx; + + // push to position chain + int afterIdx = entry->NextNear; + Metadata* after = &_spareMemory[-afterIdx]; + newEntry->NextNear = afterIdx; + after->PreviousNear = newEntryIdx; + entry->NextNear = newEntryIdx; + newEntry->PreviousNear = entryIdx; + } + h = new Handle(entry, _basePtr); return true; } - private bool TryGetNewMetadata( - out Metadata* meta - ) - { - ulong spareSize = _spareMemory->Size; - if (spareSize < Metadata.SizeOf) - { - meta = null; - return false; - } - _metadataEnd++; - meta = &_spareMemory[-_metadataEnd]; - spareSize -= Metadata.SizeOf; - _spareMemory->Size = spareSize; - return true; - } - - [StructLayout(LayoutKind.Explicit)] - public readonly struct Handle - { - - [FieldOffset(0)]internal readonly Metadata* _meta; - [FieldOffset(8)]public readonly void* Pointer; - [FieldOffset(16)]public readonly Ptr64 Size; - - internal Handle( - Metadata* meta, - Ptr64 baseAddress - ) - { - _meta = meta; - Pointer = (void*)(ulong)(meta->Position + baseAddress); - Size = meta->Size; - } - - [Obsolete("You should not create instance from default. Alloc from allocator please")] - public Handle() { } - - public ref byte this[ - int offset - ] => ref ((byte*)Pointer)[offset]; - - public Span AsSpan( - int range = -1 - ) - { - return new Span(Pointer, range < 0 ? (int)(ulong)Size : range); - } - - } - - internal enum AllocationState : short - { - - Free = 0, - Used = 1, - Split = 2, - Merged = 3 - - } - - [StructLayout(LayoutKind.Explicit, Size = 32)] - internal struct Metadata - { - - [FieldOffset(24)] private ulong PositionMask; - [FieldOffset(16)] private ulong SizeMask; - - [FieldOffset(30)] internal AllocationState State; - - [FieldOffset(0)] public int NextLevel; - [FieldOffset(4)] public int NextNear; - [FieldOffset(8)] public int PreviousLevel; - [FieldOffset(12)] public int PreviousNear; - - public Ptr64 Position - { - readonly get => PositionMask; - set => PositionMask = value; - } - - public Ptr64 Size - { - readonly get => (Ptr64)(SizeMask & Mask48Bits); - set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits); - } - - public Ptr64 SizeForSpare - { - readonly get => SizeMask; - set => SizeMask = value; - } - - public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); - - } - + #endregion } } diff --git a/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs b/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs index 12f055ce..0257ea6a 100644 --- a/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs +++ b/StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs @@ -62,7 +62,7 @@ namespace StgSharp.Mathematics.Numeric internal readonly Tlsf _allocator = allocator; - protected internal override unsafe T* BufferPointer => (T*)_handle.Pointer; + protected internal override unsafe T* BufferPointer => (T*)_handle.Address; public override void Dispose() { diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 0352fecd..4b7637f1 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -53,9 +53,9 @@ namespace StgSharp .InitializeModule(); // TestRegexAnalyse.Run(); - TestHlsf(); + // TestHlsf(); - // HlsfStressTest.Run(); + TlsfStressTest.Run(); // TestSwiss(); // TestL4Prefetch(); diff --git a/StgSharpDebug/hlsf-test.cs b/StgSharpDebug/hlsf-test.cs index cc970044..a10790c5 100644 --- a/StgSharpDebug/hlsf-test.cs +++ b/StgSharpDebug/hlsf-test.cs @@ -85,10 +85,17 @@ namespace StgSharp List batch = new(maxCount); for (int i = 0; i < maxCount; i++) { - uint sz = sizes[rnd.Next(sizes.Length)]; - batch.Add(hlsfAllocator.Alloc(sz)); - - // Console.WriteLine($"alloced:{i}"); + try + { + uint sz = sizes[rnd.Next(sizes.Length)]; + batch.Add(hlsfAllocator.Alloc(sz)); + } + catch (Exception e) + { + Console.WriteLine($"alloced:{round},{i}"); + Console.WriteLine(e); + throw; + } } for (int i = 5; i < maxCount; i += 10) { @@ -99,9 +106,16 @@ namespace StgSharp } for (int i = batch.Count - 1; i >= 0; i--) { - hlsfAllocator.Free(batch[i]); - - // Console.WriteLine($"freed:{i}"); + try + { + hlsfAllocator.Free(batch[i]); + } + catch (Exception e) + { + Console.WriteLine($"freed:{round},{i}"); + Console.WriteLine(e); + throw; + } } if (round % 100 == 0) {