diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs index 453c4d79..22227fd3 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs @@ -26,8 +26,10 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using System; +using System.ComponentModel; +using System.Diagnostics.Tracing; using System.Numerics; - +using System.Runtime.InteropServices; using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle; namespace StgSharp.HighPerformance.Memory @@ -37,7 +39,15 @@ namespace StgSharp.HighPerformance.Memory public hlsfHandle Alloc(uint size) { - ArgumentOutOfRangeException.ThrowIfGreaterThan(size, (nuint)levelSizeArray[^1] * 2); + if (size > 32U * 1024U * 1024U) + { + Entry* e = (Entry*)_entries.Allocate(); + e->Size = size; + e->Position = (nuint)NativeMemory.AlignedAlloc(size, (nuint)_align); + e->State = EntryState.Large; + return new hlsfHandle(e, size); + } + int level = GetLevelFromSize(size); int segmentIndex = DetermineSegmentIndex(size, level); @@ -63,174 +73,144 @@ namespace StgSharp.HighPerformance.Memory if (handle != null) { int requestedOffset = (segmentIndex + 1) * levelSizeArray[level]; - if ((ulong)requestedOffset > handle->Size) - { - // spare返回的块不足以切出目标大小，避免进入切片导致下溢/越界 + if (requestedOffset > handle->Size) { throw new OverflowException("Out of memory: insufficient block size returned from spare memory."); } - Entry* extra = (Entry*)_entries.Allocate(); - extra->Position = handle->Position + (nuint)requestedOffset; - extra->Size = handle->Size - (uint)requestedOffset; - extra->State = EntryState.Empty; - extra->NextNear = handle->NextNear; - extra->PreviousNear = handle; - extra->NextNear->PreviousNear = extra; - extra->PreviousNear->NextNear = extra; - SliceMemory(extra); + long remainder = handle->Size - requestedOffset; + if (remainder > 0) + { + Entry* extra = (Entry*)_entries.Allocate(); + extra->Position = handle->Position + (nuint)requestedOffset; + extra->Size = (uint)remainder; + extra->State = EntryState.Empty; + extra->NextNear = handle->NextNear; + extra->PreviousNear = handle; + extra->NextNear->PreviousNear = extra; + extra->PreviousNear->NextNear = extra; + SliceMemory(extra); + } + handle->State = EntryState.Alloc; + handle->Size = requestedOffset; + handle->Level = level; } - handle->State = EntryState.Allocated; - handle->Level = level; return new hlsfHandle(handle, size); } + /// + /// Slice the memory block into smaller blocks and add them to the appropriate buckets + /// Notice: this method is not responsible for freeing the original BucketNode + /// + /// + /// + /// private void SliceMemory(Entry* e) { - Entry* current = e; + Entry* cur = e->PreviousNear; Entry* next = e->NextNear; - ulong originOffset = e->Position - (ulong)m_Buffer; - ulong finalOffset = originOffset; - ulong end = e->Position + e->Size; - ulong endOffset = end - (ulong)m_Buffer; - ulong remainSize = e->Size; - - - if (finalOffset >= endOffset || remainSize == 0) { - return; - } - - // get level of left edge - int minLevel = int.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0); - int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel; + long remain = e->Size; + long offset = (long)e->Position - (long)m_Buffer; + long end = offset + remain; + _entries.Free((nuint)e); + int i = (BitOperations.TrailingZeroCount(remain) - 6) / 2; + i = int.Min(10, i); for (; i < levelSizeArray.Length; i++) { - int levelSize = levelSizeArray[i]; - ulong levelLong = (ulong)levelSize; - ulong levelUpper = levelLong * 4; - - // smallest board that is larger than finalOffset - ulong board = (finalOffset + levelUpper - 1) / levelUpper * levelUpper; - long span = (long)ulong.Min(board, endOffset) - (long)finalOffset; - int blockCount = (span <= 0) ? 0 : (int)(span / (long)levelLong); - if (blockCount == 0) + int size = levelSizeArray[i]; + int upper = size * 4; + long edge = (offset + upper - 1) / upper * upper; + int count = (int)((long.Min(edge, end) - offset) / size); + if (count > 0) { - continue; - } - int s = blockCount * levelSize; + int newSize = count * size; - // create and init new entry - Entry* newEntry = (Entry*)_entries.Allocate(); - newEntry->State = EntryState.Empty; - newEntry->Position = (nuint)m_Buffer + (nuint)finalOffset; - newEntry->Size = (uint)s; - BucketNode* bucket = (BucketNode*)newEntry->Position; - bucket->EntryRef = newEntry; - newEntry->Level = i; - - // link new entry - newEntry->PreviousNear = current; - newEntry->NextNear = next; - current->NextNear = newEntry; - next->PreviousNear = newEntry; - - // set new entry - current = newEntry; - - PushLevel(i, Math.Min(blockCount - 1, 2), bucket); - - finalOffset += (ulong)s; - remainSize -= (ulong)s; - if (remainSize <= (ulong)levelSize) - { - break; - } - if (remainSize == 0) { - return; - } - } - - if (i >= 10) - { - int levelSize = levelSizeArray[^1] * 4; - ulong levelLong = (ulong)levelSize; - ulong board = endOffset / levelLong * levelLong; - long span = (long)board - (long)finalOffset; - int blockCount = span > 0 ? (int)(span / (long)levelSize) : 0; - if (blockCount > 0) - { - int s = blockCount * levelSize; Entry* newEntry = (Entry*)_entries.Allocate(); - newEntry->State = EntryState.Empty; - newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); - newEntry->Size = (uint)s; - BucketNode* bucket = (BucketNode*)newEntry->Position; - bucket->EntryRef = newEntry; - newEntry->Level = 10; - - // link new entry - newEntry->PreviousNear = current; + newEntry->PreviousNear = cur; newEntry->NextNear = next; - current->NextNear = newEntry; - if (next != null) { - next->PreviousNear = newEntry; - } - + cur->NextNear = newEntry; + next->PreviousNear = newEntry; + newEntry->Position = (nuint)offset + (nuint)m_Buffer; + newEntry->State = EntryState.Empty; + newEntry->Size = (uint)newSize; newEntry->Level = i; - PushLevel(i, Math.Min(blockCount - 1, 2), bucket); - finalOffset += (ulong)s; - remainSize -= (ulong)s; + BucketNode* b = (BucketNode*)newEntry->Position; + b->EntryRef = newEntry; + PushLevel(i, count - 1, b); + offset += newSize; + remain -= (long)newSize; + cur = newEntry; } - if (remainSize == 0) { + if (remain <= 0) { return; } } - for (i -= 1; i >= 0; i--) + if (i > MaxLevel) { - int levelSize = levelSizeArray[i]; - ulong levelLong = (ulong)levelSize; - ulong board = endOffset / levelLong * levelLong; - long span = (long)board - (long)finalOffset; - int blockCount = span > 0 ? (int)(span / levelSize) : 0; - if (blockCount <= 0) + int size = levelSizeArray[^1]; + int upper = size * 4; + long edge = end / upper * upper; + int count = (int)((long.Min(edge, end) - offset) / upper); + if (count > 0) { - continue; - } - int s = blockCount * levelSize; - System.Diagnostics.Debug.Assert(finalOffset + (ulong)s <= endOffset); + int newSize = count * upper; - // create and init new entry - Entry* newEntry = (Entry*)_entries.Allocate(); - newEntry->State = EntryState.Empty; - newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset); - newEntry->Size = (uint)s; - BucketNode* bucket = (BucketNode*)newEntry->Position; - bucket->EntryRef = newEntry; - - // link new entry - newEntry->PreviousNear = current; - newEntry->NextNear = next; - current->NextNear = newEntry; - if (next != null) { + Entry* newEntry = (Entry*)_entries.Allocate(); + newEntry->PreviousNear = cur; + newEntry->NextNear = next; + cur->NextNear = newEntry; next->PreviousNear = newEntry; - } + newEntry->Position = (nuint)offset + (nuint)m_Buffer; + newEntry->State = EntryState.Empty; + newEntry->Size = (uint)newSize; + newEntry->Level = i; - newEntry->Level = i; - PushLevel(i, Math.Min(blockCount - 1, 2), bucket); + BucketNode* b = (BucketNode*)newEntry->Position; + b->EntryRef = newEntry; + PushLevel(i, count - 1, b); + cur = newEntry; + offset += newSize; + remain -= (long)newSize; + if (remain <= 0) { + return; + } + } + } - // set new entry - current = newEntry; + for (i--; i >= 0; i--) + { + int size = levelSizeArray[i]; + long edge = end / size * size; + int count = (int)((long.Min(edge, end) - offset) / size); + if (count > 0) + { + int newSize = count * size; + + Entry* newEntry = (Entry*)_entries.Allocate(); + newEntry->PreviousNear = cur; + newEntry->NextNear = next; + cur->NextNear = newEntry; + next->PreviousNear = newEntry; + newEntry->Position = (nuint)offset + (nuint)m_Buffer; + newEntry->State = EntryState.Empty; + newEntry->Size = (uint)newSize; + newEntry->Level = i; - // update lock and data - finalOffset += (ulong)s; - remainSize -= (ulong)s; - if (remainSize == 0) - { - break; + BucketNode* b = (BucketNode*)newEntry->Position; + b->EntryRef = newEntry; + b->NextLevel = null; + b->PreviousLevel = null; + PushLevel(i, count - 1, b); + + cur = newEntry; + offset += newSize; + remain -= (long)newSize; + } + if (remain <= 0) { + return; } } - RemoveFromPositionChain(e); } /// @@ -245,51 +225,32 @@ namespace StgSharp.HighPerformance.Memory private bool TryAllocateNew16MBBlock(out Entry* handle) { nuint pos = _spareMemory->Position; - nuint bufferEnd = (nuint)m_Buffer + _size; - - if (pos >= bufferEnd || _spareMemory->Size == 0) - { - handle = null; - return false; // No spare memory available - } - - nuint desired = (nuint)levelSizeArray[^1] * 4; - nuint remain = bufferEnd - pos; - nuint blockSize = desired <= remain ? desired : remain; - if (blockSize == 0) + long remain = _spareMemory->Size; + if (remain <= 0) { handle = null; return false; } + long size = remain > 16 * 1024 * 1024L ? 16 * 1024 * 1024L : remain; - nuint newPos = pos + blockSize; - // Try to allocate new Entry from slab allocator - handle = (Entry*)_entries.Allocate(); - handle->Position = pos; - handle->Size = (uint)blockSize; - handle->Level = GetLevelFromSize((uint)blockSize); + // build new entry + Entry* e = (Entry*)_entries.Allocate(); + e->Position = pos; + e->Size = size; + Entry* prev = _spareMemory->PreviousNear; + e->NextNear = _spareMemory; + e->PreviousNear = prev; + prev->NextNear = e; + _spareMemory->PreviousNear = e; + e->State = EntryState.Empty; + e->Level = GetLevelFromSize(size); - // Update spare memory position & remaining size - _spareMemory->Position = newPos; - _spareMemory->Size = (uint)(bufferEnd - newPos); - - BucketNode* b = (BucketNode*)handle->Position; - b->EntryRef = handle; - - // update position linked list - Entry* previous = _spareMemory->PreviousNear; - handle->NextNear = _spareMemory; - handle->PreviousNear = previous; - _spareMemory->PreviousNear = handle; // Replace Volatile.Write with direct access - if (previous != null) - { - previous->NextNear = handle; - } else - { - _positionHead = handle; - } + // set fields of spare memory + _spareMemory->Size -= size; + _spareMemory->Position += (nuint)size; + handle = e; return true; } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs index c4a1b3c2..1d4080ef 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs @@ -35,16 +35,6 @@ namespace StgSharp.HighPerformance.Memory public unsafe partial class HybridLayerSegregatedFitAllocator { - #region BucketNode Operations - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsNullOrEmpty(BucketNode* node) - { - return node == null; - } - - #endregion - private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node) { if (node == null || node->EntryRef == null) { @@ -52,7 +42,7 @@ namespace StgSharp.HighPerformance.Memory } Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex; - if (_bucketHeads[index] == (BucketNode*)null) + if (_bucketHeads[index] is null) { _bucketHeads[index] = node; node->IsInBucket = true; @@ -78,10 +68,29 @@ namespace StgSharp.HighPerformance.Memory } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void RemoveFromBucket(BucketNode* node) + { + BucketNode* previous = node->PreviousLevel; + BucketNode* next = node->NextLevel; + + if (previous != null) { + previous->NextLevel = next; + } + if (next != null) { + next->PreviousLevel = previous; + } + + node->NextLevel = (BucketNode*)null; + node->PreviousLevel = (BucketNode*)null; + node->IsInBucket = false; + return; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node) { Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex; - if (_bucketHeads[index] == node) { if (TryPopLevel(levelIndex, sizeIndex, out _)) { @@ -107,7 +116,7 @@ namespace StgSharp.HighPerformance.Memory private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node) { - int index = (3 * levelIndex) + sizeIndex; + Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex; BucketNode* head = _bucketHeads[index]; @@ -139,8 +148,10 @@ namespace StgSharp.HighPerformance.Memory private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node) { Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex; - if (_bucketHeads[index] == node) { - return TryPopLevel(levelIndex, sizeIndex, out _); + if (_bucketHeads[index] == node) + { + bool ok = TryPopLevel(levelIndex, sizeIndex, out _); + return ok; } BucketNode* previous = node->PreviousLevel; @@ -157,7 +168,6 @@ namespace StgSharp.HighPerformance.Memory node->PreviousLevel = null; node->IsInBucket = false; node->EntryRef = null; - return true; } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs index 7de08a83..e90dedf8 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs @@ -33,6 +33,14 @@ using System.Threading.Tasks; namespace StgSharp.HighPerformance.Memory { + public sealed class HLSFPositionChainBreakException : Exception + { + + public HLSFPositionChainBreakException() + : base("Position chain of an HLSF allocator is broken") { } + + } + public unsafe partial class HybridLayerSegregatedFitAllocator { @@ -72,7 +80,7 @@ namespace StgSharp.HighPerformance.Memory private static string EntryToString(Entry* e) { BucketNode* b = (BucketNode*)e->Position; - return $"{(ulong)e->PreviousNear}<-- Position{(ulong)e} Bucket:{{{e->Position},{e->Level},{e->Size}}} -->{(ulong)e->NextNear}"; + return $"{(ulong)e->PreviousNear}<-- Position{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -->{(ulong)e->NextNear}"; } } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs index d1b2e027..c736d2ca 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs @@ -28,7 +28,10 @@ using System; using System.Buffers; using System.Collections.Generic; +using System.Drawing; using System.Linq; +using System.Reflection.Emit; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -45,7 +48,7 @@ namespace StgSharp.HighPerformance.Memory /// public void Collect() { - Entry* cur = _positionHead; + Entry* cur = _spareMemory->NextNear; while (cur != null && cur != _spareMemory) { if (cur->State != EntryState.Empty) @@ -53,36 +56,19 @@ namespace StgSharp.HighPerformance.Memory cur = cur->NextNear; continue; } + RemoveFromBucket(cur->Level, DetermineSegmentIndex(cur->Size, cur->Level), (BucketNode*)cur->Position); + Entry* next = cur->NextNear; - Entry* runHead = cur; - Entry* next = runHead->NextNear; - - // Remove head from bucket if currently registered - int headLevel = runHead->Level; - int headSeg = DetermineSegmentIndex(runHead->Size, headLevel); - TryRemoveFromBucket(headLevel, headSeg, (BucketNode*)runHead->Position); - - while (next != null && - next != _spareMemory && + while (next is not null && next->State == EntryState.Empty && - IsAdjacentTo(runHead, next)) + AssertAdjacentTo(cur, next)) { - int lvl = next->Level; - int seg = DetermineSegmentIndex(next->Size, lvl); - TryRemoveFromBucket(lvl, seg, (BucketNode*)next->Position); - - runHead->Size += next->Size; + RemoveFromBucket(next->Level, DetermineSegmentIndex(next->Size, next->Level), (BucketNode*)next->Position); + cur->Size += next->Size; RemoveFromPositionChain(next); - next = runHead->NextNear; + next = cur->NextNear; } - - // Recompute level for merged block and push once - runHead->Level = GetLevelFromSize(runHead->Size); - int finalLevel = runHead->Level; - int finalSeg = DetermineSegmentIndex(runHead->Size, finalLevel); - ((BucketNode*)runHead->Position)->EntryRef = runHead; // ensure consistency - PushLevel(finalLevel, finalSeg, (BucketNode*)runHead->Position); - + SliceMemory(cur); cur = next; } } @@ -99,7 +85,13 @@ namespace StgSharp.HighPerformance.Memory if (e == null) { return; } - if (e->State != EntryState.Allocated) { + if (e->State == EntryState.Large) + { + NativeMemory.AlignedFree((void*)e->Position); + _entries.Free((nuint)e); + return; + } + if (e->State != EntryState.Alloc) { return; } @@ -110,12 +102,15 @@ namespace StgSharp.HighPerformance.Memory MergeAndRemoveFromBuckets(e); // Calculate final level and segment for the merged block - uint finalSize = e->Size; + long finalSize = e->Size; int finalLevel = GetLevelFromSize(finalSize); + e->Level = finalLevel; int finalSegment = DetermineSegmentIndex(finalSize, finalLevel); // Push the merged entry back to appropriate bucket PushLevel(finalLevel, finalSegment, (BucketNode*)e->Position); + + // Console.WriteLine("one block freed"); } } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs index 5e03a9d4..79732f74 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs @@ -59,20 +59,26 @@ namespace StgSharp.HighPerformance.Memory return int.Min(blockSize, _align); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int GetLevelFromSize(long size) + { + return GetLevelFromSize((uint)size); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private int GetLevelFromSize(uint size) { /* - * Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] - * Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level) - * - * For size S, we need: 2^(6+2*level) >= S - * So: 6+2*level >= log2(S) - * Therefore: level >= (log2(S) - 6) / 2 - * - * We use hardware LZCNT instruction via BitOperations.LeadingZeroCount - * log2(S) = 31 - LeadingZeroCount(S) for 32-bit values - */ + * Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] + * Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level) + * + * For size S, we need: 2^(6+2*level) >= S + * So: 6+2*level >= log2(S) + * Therefore: level >= (log2(S) - 6) / 2 + * + * We use hardware LZCNT instruction via BitOperations.LeadingZeroCount + * log2(S) = 31 - LeadingZeroCount(S) for 32-bit values + */ int log2Size = 31 - BitOperations.LeadingZeroCount(size); // Calculate level: level = max(0, (log2Size - 6) / 2) diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs index f427d203..0ee20926 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs @@ -34,6 +34,17 @@ namespace StgSharp.HighPerformance.Memory public unsafe partial class HybridLayerSegregatedFitAllocator { + /// + /// Check if two Entries are adjacent in memory + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AssertAdjacentTo(Entry* current, Entry* other) + { + // Only a predicate; never throw here. Callers rely on false to skip merging. + return other->Position + (nuint)other->Size == current->Position || + current->Position + (nuint)current->Size == other->Position; + } + /// /// check out if a bucket is within the same higher-level boundary as currentOffset /// @@ -50,21 +61,10 @@ namespace StgSharp.HighPerformance.Memory /// Determine segment index based on size and originLevel /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int DetermineSegmentIndex(uint size, int level) + private int DetermineSegmentIndex(long size, int level) { level = int.Min(level, MaxLevel); - return (int)(size - 1) / levelSizeArray[level]; - } - - /// - /// Check if two Entries are adjacent in memory - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static bool IsAdjacentTo(Entry* current, Entry* other) - { - // Check if this immediately follows other - return other->Position + other->Size == current->Position || - current->Position + current->Size == other->Position; + return (int)((size - 1) / levelSizeArray[level]); } /// @@ -80,19 +80,16 @@ namespace StgSharp.HighPerformance.Memory return; } - Entry* result = entry; - - uint size = entry->Size; + long size = entry->Size; int originLevel = entry->Level; int segment = DetermineSegmentIndex(size, originLevel); - do { // previous direction merge Entry* p = entry->PreviousNear; if (p == null || p->State != EntryState.Empty || - !IsAdjacentTo(result, p) || + !AssertAdjacentTo(entry, p) || p->Level != entry->Level || !AssertBoundary(p, entry, originLevel)) { @@ -112,7 +109,7 @@ namespace StgSharp.HighPerformance.Memory Entry* n = entry->NextNear; if (n == null || n->State != EntryState.Empty || - !IsAdjacentTo(result, n) || + !AssertAdjacentTo(entry, n) || n->Level != entry->Level || !AssertBoundary(n, entry, originLevel)) { @@ -133,24 +130,16 @@ namespace StgSharp.HighPerformance.Memory } /// - /// Remove node from position linked list + /// Remove node from position linked list, this method does not recycle a BucketNode from + /// size bucket /// private void RemoveFromPositionChain(Entry* entry) { Entry* prev = entry->PreviousNear; Entry* next = entry->NextNear; - if (prev != null) - { - prev->NextNear = entry->NextNear; - } else - { - _positionHead = entry->NextNear; - } - - if (next != null) { - next->PreviousNear = entry->PreviousNear; - } + prev->NextNear = next; + next->PreviousNear = prev; // Clear node pointers entry->NextNear = null; @@ -164,25 +153,28 @@ namespace StgSharp.HighPerformance.Memory /// Entry structure - streamlined version (only position and state information) Stored in /// external SLAB /// - [StructLayout(LayoutKind.Explicit, Size = 36)] + [StructLayout(LayoutKind.Explicit, Size = 40)] internal struct Entry { - [FieldOffset(8)] internal Entry* NextNear; // Position linked list: next pointer - [FieldOffset(16)] internal Entry* PreviousNear; // Position linked list: previous pointer + [FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer + [FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer - [FieldOffset(28)] internal int _Level; // Level of the block in the allocator - [FieldOffset(24)] internal uint Size; // Block size information - [FieldOffset(32)] public EntryState State; - [FieldOffset(0)] public nuint Position; // Memory block position + [FieldOffset(24)] internal int _Level; // Level of the block in the allocator + [FieldOffset(16)] internal long Size; // Block size information + [FieldOffset(36)] public EntryState State; + [FieldOffset(28)] public nuint Position; // Memory block position public int Level { + [MethodImpl(MethodImplOptions.AggressiveInlining)] get => _Level; + [MethodImpl(MethodImplOptions.AggressiveInlining)] set { - if (value > 15 || value < 0) { - throw new InvalidOperationException(); + int size = 64 * (1 << (value * 2)); + if (Size / size > 3 && Size < 64 * 1024 * 1024L) { + throw new HLSFPositionChainBreakException(); } _Level = value; } @@ -195,7 +187,8 @@ namespace StgSharp.HighPerformance.Memory Spare, Empty, - Allocated + Alloc, + Large } diff --git a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs index 7935aa87..8d76ff36 100644 --- a/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs +++ b/StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs @@ -39,9 +39,7 @@ namespace StgSharp.HighPerformance.Memory // Bucket system field - directly in main allocator class private BucketNode*[] _bucketHeads; - // Head of the position linked list (first Entry in memory order) - private Entry* _positionHead; - + // _spareMemory is also the beginning position of allocator private Entry* _spareMemory; private readonly byte* m_Buffer; private bool disposedValue; @@ -63,8 +61,8 @@ namespace StgSharp.HighPerformance.Memory _spareMemory = (Entry*)_entries.Allocate(); _spareMemory->State = EntryState.Spare; - _spareMemory->PreviousNear = null; - _spareMemory->NextNear = null; + _spareMemory->PreviousNear = _spareMemory; + _spareMemory->NextNear = _spareMemory; _spareMemory->Position = (nuint)m_Buffer; _spareMemory->Size = (uint)byteSize; } @@ -73,19 +71,19 @@ namespace StgSharp.HighPerformance.Memory { if (align < 16 || (align & (align - 1)) != 0) { throw new ArgumentException( - "Alignment must be a power of two and at least 8 bytes."); + "Alignment must be a power of two and at least 16 bytes."); } _entries = SlabAllocator.Create(64, SlabBufferLayout.Chunked); m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align); _align = align; _size = byteSize; - _bucketHeads = new BucketNode*[levelSizeArray.Length * 3]; + _bucketHeads = new BucketNode*[levelSizeArray.Length * 3 + 1]; _spareMemory = (Entry*)_entries.Allocate(); _spareMemory->State = EntryState.Empty; - _spareMemory->PreviousNear = null; - _spareMemory->NextNear = null; + _spareMemory->PreviousNear = _spareMemory; + _spareMemory->NextNear = _spareMemory; _spareMemory->Position = (nuint)m_Buffer; _spareMemory->Size = (uint)byteSize; } diff --git a/StgSharpDebug/Properties/launchSettings.json b/StgSharpDebug/Properties/launchSettings.json index 32cd25bd..e5f4064d 100644 --- a/StgSharpDebug/Properties/launchSettings.json +++ b/StgSharpDebug/Properties/launchSettings.json @@ -2,7 +2,8 @@ "profiles": { "StgSharpDebug": { "commandName": "Project", - "nativeDebugging": true + "nativeDebugging": true, + "hotReloadEnabled": false } } } \ No newline at end of file