youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hlsf goes bad

Browse Source
This commit is contained in:
Youmu Liu
2026-05-08 18:09:12 +08:00
parent 8aa319a120
commit d1aca8f745
47 changed files with 5020 additions and 446 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+25 -11
View File
@@ -53,13 +53,13 @@ namespace StgSharp.HighPerformance.Memory
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int index = int.Min((3 * level) + segment, _bucketHeads.Length);
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(i, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
if ((i >= _bucketHeads.Length) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException($"Out of memory: unable to satisfy request of {size} B (aligned {requestedOffset} B) from existing buckets or spare memory.");
}
if (entry != null)
{
@@ -96,13 +96,13 @@ namespace StgSharp.HighPerformance.Memory
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int index = int.Min((3 * level) + segment, _bucketHeads.Length);
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(i, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
if ((i >= _bucketHeads.Length) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException($"Out of memory: unable to satisfy request of {size} B (aligned {requestedOffset} B) from existing buckets or spare memory.");
}
if (entry != null)
@@ -138,10 +138,24 @@ namespace StgSharp.HighPerformance.Memory
{
Entry* next = e;
nuint remain = sizeToSlice; // remained size to slice
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
nuint position = e->Position - sizeToSlice; // position of buffer to slice
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
int i = GetBucketIndexOnce(sizeToSlice);
Entry* entry = (Entry*)_entries.Allocate();
SetEntryLevel(entry, position, remain);
entry->State = EntryState.Empty;
BucketNode* bucketNode = (BucketNode*)entry->Position;
bucketNode->EntryRef = entry;
PushNodeToBucket(i, bucketNode);
Entry* prev = next->PreviousNear;
prev->NextNear = entry;
entry->PreviousNear = prev;
entry->NextNear = next;
next->PreviousNear = entry;
}
/// <summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+12 -11
View File
@@ -34,6 +34,12 @@ namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/*
* All operations with style of (level, segment)
* DO NOT assume that the level are valid.
* They are just used to calculate the bucket index,
* which may exceed the range of _bucketHeads.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void PushBucketToLevel(
@@ -46,7 +52,7 @@ namespace StgSharp.HighPerformance.Memory
return;
}
int index = (3 * levelIndex) + sizeIndex;
int index = Math.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
PushNodeToBucket(index, node);
}
@@ -55,7 +61,7 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* node
)
{
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
index = Math.Min(index, _bucketHeads.Length - 1);
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
@@ -69,15 +75,12 @@ namespace StgSharp.HighPerformance.Memory
{
BucketNode* head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = head;
node->IsInBucket = true;
_bucketHeads[index] = node;
// Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}");
if (headNode != null)
{
headNode->PreviousLevel = node;
}
head->PreviousLevel = node;
return;
}
}
@@ -89,8 +92,7 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* node
)
{
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
int index = Math.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
if (_bucketHeads[index] == node)
{
if (TryPopBucket(index, out _)) {
@@ -157,8 +159,7 @@ namespace StgSharp.HighPerformance.Memory
out BucketNode* node
)
{
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? (_bucketHeads.Length - 1) : index;
int index = int.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
return TryPopBucket(index, out node);
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+15 -19
View File
@@ -74,23 +74,25 @@ namespace StgSharp.HighPerformance.Memory
{
size += cur->Size;
Entry* next = cur->NextNear;
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromBucket(cur->Level, GetSegmentFromLevel(cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
if (size > 0) {
if (size > 0)
{
OrganizeMemory(cur, size);
size = 0;
}
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(
HlsfHandle handle,
FreePolicy collectPolicy = FreePolicy.NoCollect
@@ -108,16 +110,12 @@ namespace StgSharp.HighPerformance.Memory
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
(int finalLevel, int finalSegment) = GetIndexPairOnce(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
@@ -145,16 +143,14 @@ namespace StgSharp.HighPerformance.Memory
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
/**/
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
int finalSegment = GetSegmentFromLevel(finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs
+15 -2
View File
@@ -62,7 +62,7 @@ namespace StgSharp.HighPerformance.Memory
level = Math.Min(level, _maxLevel);
uint levelSize = 1u << (log2Size & (~1));
int segment = ((int)(size / levelSize)) - 1;
return (level * 3) + segment;
return Math.Min((level * 3) + segment, _bucketHeads.Length - 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -82,6 +82,19 @@ namespace StgSharp.HighPerformance.Memory
return (level, segment);
}
/// <summary>
/// NOTICE: returned level may exceed maxLevel, caller is responsible for ensuring the
/// returned level is within bounds. This method is designed for performance and does not
/// perform bounds checking.
/// </summary>
/// <param name="size">
/// Size of the memory block to determine level for. This method does not assume size is
/// already aligned
/// </param>
/// <returns>
/// Level corresponding to the given size. Level is determined based on the formula: level
/// >= (log2(S) - 6) / 2
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevelFromSize(
nuint size
@@ -104,7 +117,7 @@ namespace StgSharp.HighPerformance.Memory
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return Math.Min(level, _maxLevel);
return level;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+13 -11
View File
@@ -42,13 +42,12 @@ namespace StgSharp.HighPerformance.Memory
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(
Entry* current,
Entry* other
Entry* front,
Entry* back
)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return (other->Position + other->Size == current->Position) ||
(current->Position + current->Size == other->Position);
return front->Position + front->Size == back->Position;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -83,7 +82,8 @@ namespace StgSharp.HighPerformance.Memory
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(
Entry* entry
Entry* entry,
int maxScanRange = 4
)
{
if (entry == null) {
@@ -94,7 +94,8 @@ namespace StgSharp.HighPerformance.Memory
nuint size = entry->Size;
int originLevel = entry->Level;
int segment = GetSegmentFromLevel(size, originLevel);
do
for (int i = 0; i < maxScanRange; i++)
{
// previous direction merge
Entry* p = entry->PreviousNear;
@@ -112,9 +113,9 @@ namespace StgSharp.HighPerformance.Memory
entry->Position = p->Position;
RemoveFromBucket(originLevel, GetSegmentFromLevel(p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
}
do
for (int i = 0; i < maxScanRange; i++)
{
// next direction merge
Entry* n = entry->NextNear;
@@ -122,7 +123,7 @@ namespace StgSharp.HighPerformance.Memory
(n == _spareMemory) ||
(n->State != EntryState.Empty) ||
(n->Level != entry->Level) ||
!AssertAdjacentTo(n, entry))
!AssertAdjacentTo(entry, n))
{
break;
}
@@ -131,8 +132,9 @@ namespace StgSharp.HighPerformance.Memory
entry->Size += n->Size;
RemoveFromBucket(originLevel, GetSegmentFromLevel(n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
}
// originLevel = int.Min(9, originLevel);
entry->Level = GetLevelFromSize(entry->Size);
BucketNode* bucket = (BucketNode*)entry->Position;
StgSharp.Common/HighPerformance/Memory/L4.Debug.cs
+1 -1
View File
@@ -39,7 +39,7 @@ namespace StgSharp.HighPerformance.Memory
public void DebugAgeAll()
{
AgeAll();
// AgeAll();
}
public bool DebugContains(
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+5 -5
View File
@@ -109,11 +109,11 @@ namespace StgSharp.HighPerformance.Memory
*
* This is safe only with the current L4 allocation/layout guarantees.
*/
short predictCount = Volatile.Read(ref (*_predictCount)[index]);
short mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (short.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_predictCount)[index]), -decrement);
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_mapCount)[index]), -decrement);
int predictCount = Volatile.Read(ref (*_predictCount)[index]);
int mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref (*_predictCount)[index], -decrement);
_ = Interlocked.Add(ref (*_mapCount)[index], -decrement);
}
(*_predictCount)[index]++;
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs
+24 -28
View File
@@ -56,29 +56,25 @@ namespace StgSharp.HighPerformance.Memory
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para> While SLAB
/// /// /// /// allocators themselves may be designed for single-threaded use,
/// allocated /// objects /// are /// frequently shared across multiple threads in
/// real-world /// applications. Due to /// the /// special implementation of
/// expandable concurrent /// sequential SLAB allocators, /// they rely /// on
/// internal expansion locks to /// ensure that memory buffer expansions /// (which
/// involve /// copying the entire /// buffer to a new location) do not occur while
/// /// other threads are /// reading /// from allocated regions.</para>
/// <para><strong> /// Sequential Layout Limitations: /// /// </strong><br />
/// Sequential layout SLAB allocators /// cannot provide a single- /// threaded ///
/// version because:<list /// type="bullet"><item><description> They use ///
/// contiguous memory /// layout that requires /// buffer expansion when capacity is
/// /// exceeded /// /// </description></item><item><description> Buffer expansion
/// involves /// copying all /// /// existing data to a new memory /// location
/// </description></item><item><description>/// /// Without proper ///
/// synchronization, concurrent access to allocated objects during /// /// expansion
/// /// would cause memory corruption </description></item><item><description> The ///
/// /// /// expansion lock mechanism is essential even if allocation itself is
/// single- /// /// /// threaded</description></item></list></para> <para><strong>
/// Chunked Layout: /// /// /// </strong><br /> Chunked layout allocators can
/// optionally provide single-threaded /// /// /// versions since they don't
/// require buffer expansion and copying. Each /// chunk is /// /// allocated
/// independently, avoiding the memory relocation issues /// present in ///
/// sequential /// layouts.</para>
/// allocators themselves may be designed for single-threaded use, allocated objects are
/// frequently shared across multiple threads in real-world applications. Due to the
/// special implementation of expandable concurrent sequential SLAB allocators, they rely
/// on internal expansion locks to ensure that memory buffer expansions (which involve
/// copying the entire buffer to a new location) do not occur while other threads are
/// reading from allocated regions.</para> <para><strong>Sequential Layout
/// Limitations:</strong><br /> Sequential layout SLAB allocators cannot provide a single-
/// threaded version because:<list type="bullet"><item><description>They use contiguous
/// memory layout that requires buffer expansion when capacity is
/// exceeded</description></item><item><description>Buffer expansion involves copying all
/// existing data to a new memory location</description></item><item><description> Without
/// proper synchronization, concurrent access to allocated objects during synchronization,
/// concurrent access to allocated objects during expansion would cause memory corruption
/// </description></item><item><description> The expansion lock mechanism is essential
/// even if allocation itself is single- threaded</description></item></list></para>
/// <para><strong> Chunked Layout: </strong><br /> Chunked layout allocators can
/// optionally provide single-threaded versions since they don't require buffer expansion
/// and copying. Each chunk is allocated independently, avoiding the memory relocation
/// issues present in sequential layouts.</para>
/// </remarks>
public static SlabAllocator<T> Create<T>(
nuint count,
@@ -89,12 +85,12 @@ namespace StgSharp.HighPerformance.Memory
return layout switch
{
SlabBufferLayout.Sequential => concurrentSupport ?
(new ConcurrentSequentialSlabAllocator<T>(count)) :
(throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
new ConcurrentSequentialSlabAllocator<T>(count) :
throw (new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
SlabBufferLayout.Chunked => concurrentSupport ?
(new ConcurrentChunkedSlabAllocator<T>(count)) :
(new ChunkedSlabAllocator<T>(count)),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
new ConcurrentChunkedSlabAllocator<T>(count) :
new ChunkedSlabAllocator<T>(count),
_ => throw (new ArgumentOutOfRangeException(nameof(layout), layout, null))
};
}