diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs index be6d86df..e3c44d9f 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs @@ -80,100 +80,151 @@ namespace StgSharp.HighPerformance.Memory private void SliceMemory(Entry* e, int offset) { Entry* current = e; - Entry* n = e->NextNear; - Entry* next = e; - + Entry* next = e->NextNear; ulong originOffset = e->Position - (ulong)m_Buffer; ulong finalOffset = originOffset + (ulong)offset; ulong end = e->Position + e->Size; ulong endOffset = end - (ulong)m_Buffer; - int remainSize = (int)(e->Size - offset); + ulong remainSize = (ulong)e->Size - (ulong)offset; // get level of left edge int minLevel = int.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0); int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel; - - for (; i < levelSizeArray.Length; i++) + try { - int levelSize = levelSizeArray[i]; - ulong levelLong = (ulong)levelSize; - ulong levelUpper = levelLong * 4; - ulong board = (finalOffset + levelUpper - 1) / levelUpper * levelUpper; - int blockCount = (int)((board - ulong.Min(finalOffset, endOffset)) / levelLong); - if (blockCount == 0) + for (; i < levelSizeArray.Length; i++) { - continue; + int levelSize = levelSizeArray[i]; + ulong levelLong = (ulong)levelSize; + ulong levelUpper = levelLong * 4; + + // smallest board that is larger than finalOffset + ulong board = (finalOffset + levelUpper - 1) / levelUpper * levelUpper; + int blockCount = (int)((long)ulong.Min(board, endOffset) - (long)finalOffset) / (int)levelLong; + if (blockCount == 0) + { + continue; + } + int s = blockCount * levelSize; + + // create and init new entry + Entry* newEntry = (Entry*)_entries.Allocate(); + newEntry->State = EntryState.Empty; + newEntry->Position = (nuint)m_Buffer + (nuint)finalOffset; + newEntry->Size = (uint)s; + BucketNode* bucket = (BucketNode*)newEntry->Position; + bucket->EntryRef = newEntry; + + // link new entry + newEntry->PreviousNear = current; + newEntry->NextNear = next; + current->NextNear = newEntry; + next->PreviousNear = newEntry; + + newEntry->Level = i; + PushLevel(i, Math.Min(blockCount - 1, 2), bucket); + + // set new entry + // Entry* old = current; + current = newEntry; + + finalOffset += (ulong)s; + remainSize -= (ulong)s; + if (remainSize <= (ulong)levelSize) + { + break; + } + if (remainSize <= 0) { + return; + } } - int s = blockCount * levelSize; - // create and init new entry - Entry* newEntry = (Entry*)_entries.Allocate(); - newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); - newEntry->Size = (uint)s; - BucketNode* bucket = (BucketNode*)newEntry->Position; - bucket->EntryRef = newEntry; + if (i >= 10) + { + int levelSize = levelSizeArray[^1] * 4; + ulong levelLong = (ulong)levelSize; + ulong board = endOffset / levelLong * levelLong; + int blockCount = (int)((board - finalOffset) / (ulong)levelSize); + if (blockCount > 0) + { + int s = blockCount * levelSize; + Entry* newEntry = (Entry*)_entries.Allocate(); + newEntry->State = EntryState.Empty; + newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); + newEntry->Size = (uint)s; + BucketNode* bucket = (BucketNode*)newEntry->Position; + bucket->EntryRef = newEntry; - // link new entry - newEntry->PreviousNear = current; - newEntry->NextNear = n; - current->NextNear = newEntry; - next->PreviousNear = newEntry; - newEntry->Level = i; - PushLevel(i, blockCount - 1, bucket); + // link new entry + newEntry->PreviousNear = current; + newEntry->NextNear = next; + current->NextNear = newEntry; + if (next != null) { + next->PreviousNear = newEntry; + } - // set new entry - Entry* old = current; - current = newEntry; + newEntry->Level = i; + PushLevel(i, Math.Min(blockCount - 1, 2), bucket); - finalOffset += (ulong)s; - remainSize -= s; - if (remainSize <= 0) { - return; + // update lock and data + finalOffset += (ulong)s; + remainSize -= (ulong)s; + } + if (remainSize <= 0) { + return; + } + } + + for (i -= 1; i >= 0; i--) + { + int levelSize = levelSizeArray[i]; + ulong levelLong = (ulong)levelSize; + ulong board = endOffset / levelLong * levelLong; + int blockCount = (int)(board - finalOffset) / levelSize; + if (blockCount <= 0) + { + continue; + } + int s = blockCount * levelSize; + System.Diagnostics.Debug.Assert(finalOffset + (ulong)s <= endOffset); + + // create and init new entry + Entry* newEntry = (Entry*)_entries.Allocate(); + newEntry->State = EntryState.Empty; + newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); + newEntry->Size = (uint)s; + BucketNode* bucket = (BucketNode*)newEntry->Position; + bucket->EntryRef = newEntry; + + // link new entry + newEntry->PreviousNear = current; + newEntry->NextNear = next; + current->NextNear = newEntry; + if (next != null) { + next->PreviousNear = newEntry; + } + + newEntry->Level = i; + PushLevel(i, Math.Min(blockCount - 1, 2), bucket); + + // set new entry + // Entry* old = current; + current = newEntry; + + // update lock and data + finalOffset += (ulong)s; + remainSize -= (ulong)s; + if (remainSize <= 0) + { + break; + } } } - - for (i -= 1; i >= 0; i--) + finally { - int levelSize = levelSizeArray[i]; - ulong levelLong = (ulong)levelSize; - ulong board = endOffset / levelLong * levelLong; - int blockCount = (int)((board - finalOffset) / (ulong)levelSize); - if (blockCount == 0) - { - continue; - } - int s = blockCount * levelSize; - - // create and init new entry - Entry* newEntry = (Entry*)_entries.Allocate(); - newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); - newEntry->Size = (uint)s; - BucketNode* bucket = (BucketNode*)newEntry->Position; - bucket->EntryRef = newEntry; - - // link new entry - newEntry->PreviousNear = current; - newEntry->NextNear = n; - current->NextNear = newEntry; - if (next != null) { - next->PreviousNear = newEntry; - } - - newEntry->Level = i; - PushLevel(i, blockCount - 1, bucket); - - // set new entry - Entry* old = current; - current = newEntry; - - // update lock and data - finalOffset += (ulong)s; - remainSize -= s; - if (remainSize <= 0) - { - break; - } + e->Size = (uint)(offset); + e->Level = GetLevelFromSize((uint)offset); } } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs index 97acc2f0..2b2fa7ef 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs @@ -66,6 +66,7 @@ namespace StgSharp.HighPerformance.Memory node->NextLevel = head; BucketNode* headNode = head; node->IsInBucket = true; + _bucketHeads[index] = node; // Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}"); if (headNode != null) diff --git a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs index 0966182a..5c092ef5 100644 --- a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs +++ b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs @@ -35,7 +35,7 @@ using System.Threading; namespace StgSharp.HighPerformance.Memory { - internal sealed unsafe class ChunkedSlabAllocator : SlabAllocator where T: unmanaged + internal sealed unsafe class ChunkedSlabAllocator : SlabAllocator where T : unmanaged { private BufferStack _buffers; @@ -48,6 +48,7 @@ namespace StgSharp.HighPerformance.Memory public ChunkedSlabAllocator(nuint count) { _currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize); + _currentCapacity = count; int initCount = count switch { @@ -60,6 +61,7 @@ namespace StgSharp.HighPerformance.Memory span[i] = _currentBuffer + (nuint)(i * _elementSize); } _buffers = new(4); + _buffers.Push(_currentBuffer); _recycle = BufferStackBuilder.Create(span); _currentCapacity = count; _currentIndex = (nuint)initCount; @@ -70,26 +72,27 @@ namespace StgSharp.HighPerformance.Memory if (_recycle.TryPop(out nuint handle)) { return handle; } - nuint bufferHandle = _currentBuffer; + + // nuint bufferHandle = _currentBuffer; nuint curIndex = _currentIndex; if (curIndex >= _currentCapacity) { nuint cap = _currentCapacity; nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize); - _buffers.Push(bufferHandle); + _buffers.Push(newBuffer); _currentBuffer = newBuffer; _currentIndex = 0; } curIndex = _currentIndex; - _ = Interlocked.Add( - ref Unsafe.As(ref _currentIndex), (ulong)_elementSize); + _currentIndex += 1; return _currentBuffer + ((nuint)_elementSize * curIndex); } public override void Dispose() { - NativeMemory.Free((void*)_currentBuffer); - while (_buffers.TryPop(out nuint buffer)) { + // NativeMemory.Free((void*)_currentBuffer); + while (_buffers.TryPop(out nuint buffer)) + { NativeMemory.Free((void*)buffer); } _buffers.Dispose(); diff --git a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs index 94d5833b..340f1a93 100644 --- a/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs +++ b/StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs @@ -29,7 +29,7 @@ using System; namespace StgSharp.HighPerformance.Memory { - public abstract class SlabAllocator : IDisposable where T: unmanaged + public abstract class SlabAllocator : IDisposable where T : unmanaged { public abstract nuint Allocate(); @@ -56,29 +56,32 @@ namespace StgSharp.HighPerformance.Memory /// Thrown when requesting unsupported configurations. /// /// - /// Important Threading Considerations: While SLAB + /// Important Threading Considerations: While SLAB /// /// allocators themselves may be designed for single-threaded use, allocated objects /// are /// frequently shared across multiple threads in real-world applications. Due to /// the /// special implementation of expandable concurrent sequential SLAB allocators, /// they rely /// on internal expansion locks to ensure that memory buffer expansions /// (which involve /// copying the entire buffer to a new location) do not occur while - /// other threads are /// reading from allocated regions. - /// Sequential Layout Limitations: ///
Sequential layout SLAB - /// allocators cannot provide a single-threaded /// version because:They use contiguous memory /// layout that requires - /// buffer expansion when capacity is exceeded - /// ///Buffer expansion involves copying all /// - /// existing data to a new memory location /// + /// other threads are /// reading from allocated regions. + /// Sequential Layout Limitations: ///
Sequential layout SLAB allocators + /// cannot provide a single-threaded /// version because: They use contiguous memory /// layout that + /// requires buffer expansion when capacity is exceeded /// + /// Buffer expansion involves copying all /// + /// existing data to a new memory location /// /// Without proper synchronization, concurrent access to allocated objects during /// - /// expansion would cause memory corruptionThe + /// expansion would cause memory corruption The /// /// expansion lock mechanism is essential even if allocation itself is single- /// - /// threaded Chunked Layout: - /// ///
Chunked layout allocators can optionally provide single-threaded - /// /// versions since they don't require buffer expansion and copying. Each chunk is - /// /// allocated independently, avoiding the memory relocation issues present in - /// sequential /// layouts. + /// threaded Chunked Layout: /// + ///
Chunked layout allocators can optionally provide single-threaded /// + /// versions since they don't require buffer expansion and copying. Each chunk is /// + /// allocated independently, avoiding the memory relocation issues present in sequential + /// /// layouts. /// - public static SlabAllocator Create(nuint count, SlabBufferLayout layout, bool concurrentSupport = true) + public static SlabAllocator Create( + nuint count, + SlabBufferLayout layout, + bool concurrentSupport = true) { if (concurrentSupport) { return layout switch diff --git a/StgSharpDebug/HlsfStressTest.cs b/StgSharpDebug/HlsfStressTest.cs index c4ba409e..c5997189 100644 --- a/StgSharpDebug/HlsfStressTest.cs +++ b/StgSharpDebug/HlsfStressTest.cs @@ -25,10 +25,6 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="HlsfStressTest" (performance benchmark) -// Project: StgSharp -// ----------------------------------------------------------------------- using StgSharp.HighPerformance.Memory; using System; using System.Collections.Generic; @@ -46,7 +42,7 @@ namespace StgSharp int.TryParse(mbStr, out int mb) && mb >= 64 ? (ulong)mb * 1024UL * 1024UL : - 3UL * 1024UL * 1024UL * 1024UL; + 512UL * 1024UL * 1024UL; // lower default to 512MB private static readonly uint[] SizeSweep = { 64, 128, 192, 256, 512, 768, 1024, 2048, 3072, 4096, 8192, 12288, 16384, 32768, 49152, 65536, 131072, 196608, 262144, 524288, 786432, 1_048_576, 2_097_152, 3_145_728, 4_194_304, 8_388_608, 12_582_912, 16_777_216, 25_165_824, 33_554_432 @@ -56,42 +52,61 @@ namespace StgSharp { Console.WriteLine("=== HLSF Allocator Single-Thread Performance Benchmark ==="); Console.WriteLine($"Arena size: {ArenaBytes / (1024 * 1024)} MiB (env override: HLSF_ARENA_MB)"); - using HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes); - Warmup(allocator); Console.WriteLine(); Console.WriteLine("[1] Allocation throughput (no touch)"); - foreach (uint sz in SizeSweep) + using (HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes)) { - uint size = sz > MaxAllocSize ? MaxAllocSize : sz; - BenchAllocThroughput(allocator, size, touch:false); - GC.Collect(); - GC.WaitForPendingFinalizers(); + Warmup(allocator); + + foreach (uint sz in SizeSweep) + { + uint size = sz > MaxAllocSize ? MaxAllocSize : sz; + BenchAllocThroughput(allocator, size, touch:false); + GC.Collect(); + GC.WaitForPendingFinalizers(); + } } Console.WriteLine(); Console.WriteLine("[2] Allocation throughput (touch first byte)"); - foreach (uint sz in SizeSweep) + using (HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes)) { - uint size = sz > MaxAllocSize ? MaxAllocSize : sz; - BenchAllocThroughput(allocator, size, touch:true); - GC.Collect(); - GC.WaitForPendingFinalizers(); + Warmup(allocator); + + foreach (uint sz in SizeSweep) + { + uint size = sz > MaxAllocSize ? MaxAllocSize : sz; + BenchAllocThroughput(allocator, size, touch:true); + GC.Collect(); + GC.WaitForPendingFinalizers(); + } } - Console.WriteLine(); - Console.WriteLine("[3] Free throughput after pre-allocation (256k blocks of 1KB)"); - BenchFreeThroughput(allocator, 256 * 1024, 1024); + using (HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes)) + { + Warmup(allocator); + Console.WriteLine("[3] Free throughput after pre-allocation (256k blocks of 1KB)"); + BenchFreeThroughput(allocator, 256 * 1024, 1024); + } Console.WriteLine(); Console.WriteLine("[4] Alloc+Immediate Free pairs (5,000,000 x 256B)"); - BenchAllocFreePairs(allocator, 5_000_000, 256); + using (HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes)) + { + Warmup(allocator); + BenchAllocFreePairs(allocator, 5_000_000, 256); + } Console.WriteLine(); Console.WriteLine("[5] Fragmentation cycle (allocate ~1024MB random, free half, reallocate)"); - BenchFragmentationCycle(allocator, 1024); + using (HybridLayerSegregatedFitAllocator allocator = new HybridLayerSegregatedFitAllocator((nuint)ArenaBytes)) + { + Warmup(allocator); + BenchFragmentationCycle(allocator, 512); // target within 500MB arena + } Console.WriteLine(); Console.WriteLine("=== Benchmark Complete ==="); } diff --git a/StgSharpDebug/Program.cs b/StgSharpDebug/Program.cs index 34018b0b..bfac26f9 100644 --- a/StgSharpDebug/Program.cs +++ b/StgSharpDebug/Program.cs @@ -25,30 +25,8 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- -using StgSharp.Collections; -using StgSharp.Controlling.UsrActivity; -using StgSharp.Geometries; -using StgSharp.Graphics; -using StgSharp.HighPerformance; -using StgSharp.HighPerformance.Memory; using StgSharp.Internal; -using StgSharp.Mathematics; -using StgSharp.Mathematics.Graphic; using StgSharp.Mathematics.Numeric; -using StgSharp.Model.Step; -using StgSharp.MVVM; -using StgSharp.MVVM.ViewModel; -using StgSharp.Script; -using StgSharp.Script.Express; - -using System; -using System.Collections.Concurrent; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; -using System.Threading; -using System.Xml.XPath; namespace StgSharp { diff --git a/StgSharpDebug/SlabStressTest.cs b/StgSharpDebug/SlabStressTest.cs new file mode 100644 index 00000000..9fc70109 --- /dev/null +++ b/StgSharpDebug/SlabStressTest.cs @@ -0,0 +1,82 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="SlabStressTest" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.HighPerformance.Memory; +using System; +using System.Diagnostics; + +namespace StgSharp +{ + internal static class SlabStressTest + { + + // Exercise the Nonconcurrent Chunked SLAB used by HLSF (allocation/free only) + public static unsafe void Run() + { + Console.WriteLine("=== SLAB Allocator (Chunked) Single-Thread Stress Test ==="); + + // Use SLAB with element size of pointer (nuint) to simulate Entry handles + SlabAllocator slab = SlabAllocator.Create(64, SlabBufferLayout.Chunked, false); + + int cycles = 10; + int allocsPerCycle = 200_000; // large enough to force internal growth + + Stopwatch sw = new Stopwatch(); + + for (int c = 0; c < cycles; c++) + { + Console.WriteLine($"--- Cycle {c+1} ---"); + nuint[] ptrs = new nuint[allocsPerCycle]; + + sw.Restart(); + for (int i = 0; i < allocsPerCycle; i++) { + ptrs[i] = slab.Allocate(); + } + sw.Stop(); + Console.WriteLine($"Allocate: {allocsPerCycle} in {sw.Elapsed.TotalSeconds:0.00}s"); + + sw.Restart(); + for (int i = 0; i < allocsPerCycle; i++) { + slab.Free(ptrs[i]); + } + sw.Stop(); + Console.WriteLine($"Free: {allocsPerCycle} in {sw.Elapsed.TotalSeconds:0.00}s"); + } + + slab.Dispose(); + Console.WriteLine("=== SLAB Stress Complete ==="); + } + + internal unsafe struct BunchOfBytes + { + + public fixed byte Data[128]; + + } + + } +}