youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hlsf gong on rewrite again

Browse Source
This commit is contained in:
Youmu Liu
2025-11-27 20:13:09 +08:00
parent 83537c7be7
commit a3570bf20a
7 changed files with 267 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+142 -126
View File
@@ -60,134 +60,112 @@ namespace StgSharp.HighPerformance.Memory
}
}
if (handle != null) {
SliceMemory(handle, (segmentIndex + 1) * levelSizeArray[level]);
if (handle != null)
{
int requestedOffset = (segmentIndex + 1) * levelSizeArray[level];
if ((ulong)requestedOffset > handle->Size)
{
// spare返回的块不足以切出目标大小，避免进入切片导致下溢/越界
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
Entry* extra = (Entry*)_entries.Allocate();
extra->Position = handle->Position + (nuint)requestedOffset;
extra->Size = handle->Size - (uint)requestedOffset;
extra->State = EntryState.Empty;
extra->NextNear = handle->NextNear;
extra->PreviousNear = handle;
extra->NextNear->PreviousNear = extra;
extra->PreviousNear->NextNear = extra;
SliceMemory(extra);
}
handle->State = EntryState.Allocated;
handle->Level = level;
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset)
private void SliceMemory(Entry* e)
{
Entry* current = e;
Entry* next = e->NextNear;
ulong originOffset = e->Position - (ulong)m_Buffer;
ulong finalOffset = originOffset + (ulong)offset;
ulong finalOffset = originOffset;
ulong end = e->Position + e->Size;
ulong endOffset = end - (ulong)m_Buffer;
ulong remainSize = (ulong)e->Size - (ulong)offset;
ulong remainSize = e->Size;
if (finalOffset >= endOffset || remainSize == 0) {
return;
}
// get level of left edge
int minLevel = int.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0);
int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel;
try
for (; i < levelSizeArray.Length; i++)
{
for (; i < levelSizeArray.Length; i++)
int levelSize = levelSizeArray[i];
ulong levelLong = (ulong)levelSize;
ulong levelUpper = levelLong * 4;
// smallest board that is larger than finalOffset
ulong board = (finalOffset + levelUpper - 1) / levelUpper * levelUpper;
long span = (long)ulong.Min(board, endOffset) - (long)finalOffset;
int blockCount = (span <= 0) ? 0 : (int)(span / (long)levelLong);
if (blockCount == 0)
{
int levelSize = levelSizeArray[i];
ulong levelLong = (ulong)levelSize;
ulong levelUpper = levelLong * 4;
// smallest board that is larger than finalOffset
ulong board = (finalOffset + levelUpper - 1) / levelUpper * levelUpper;
int blockCount = (int)((long)ulong.Min(board, endOffset) - (long)finalOffset) / (int)levelLong;
if (blockCount == 0)
{
continue;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->State = EntryState.Empty;
newEntry->Position = (nuint)m_Buffer + (nuint)finalOffset;
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
// link new entry
newEntry->PreviousNear = current;
newEntry->NextNear = next;
current->NextNear = newEntry;
next->PreviousNear = newEntry;
// set new entry
// Entry* old = current;
current = newEntry;
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// Console.WriteLine($"{(ulong)current->PreviousNear}<->{(ulong)current->NextNear},\t{current->Size}");
finalOffset += (ulong)s;
remainSize -= (ulong)s;
if (remainSize <= (ulong)levelSize)
{
break;
}
if (remainSize <= 0) {
return;
}
continue;
}
int s = blockCount * levelSize;
if (i >= 10)
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->State = EntryState.Empty;
newEntry->Position = (nuint)m_Buffer + (nuint)finalOffset;
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
newEntry->Level = i;
// link new entry
newEntry->PreviousNear = current;
newEntry->NextNear = next;
current->NextNear = newEntry;
next->PreviousNear = newEntry;
// set new entry
current = newEntry;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
finalOffset += (ulong)s;
remainSize -= (ulong)s;
if (remainSize <= (ulong)levelSize)
{
int levelSize = levelSizeArray[^1] * 4;
ulong levelLong = (ulong)levelSize;
ulong board = endOffset / levelLong * levelLong;
int blockCount = (int)((board - finalOffset) / (ulong)levelSize);
if (blockCount > 0)
{
int s = blockCount * levelSize;
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->State = EntryState.Empty;
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
// link new entry
newEntry->PreviousNear = current;
newEntry->NextNear = next;
current->NextNear = newEntry;
if (next != null) {
next->PreviousNear = newEntry;
}
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// update lock and data
finalOffset += (ulong)s;
remainSize -= (ulong)s;
}
if (remainSize <= 0) {
return;
}
break;
}
if (remainSize == 0) {
return;
}
}
for (i -= 1; i >= 0; i--)
if (i >= 10)
{
int levelSize = levelSizeArray[^1] * 4;
ulong levelLong = (ulong)levelSize;
ulong board = endOffset / levelLong * levelLong;
long span = (long)board - (long)finalOffset;
int blockCount = span > 0 ? (int)(span / (long)levelSize) : 0;
if (blockCount > 0)
{
int levelSize = levelSizeArray[i];
ulong levelLong = (ulong)levelSize;
ulong board = endOffset / levelLong * levelLong;
int blockCount = (int)(board - finalOffset) / levelSize;
if (blockCount <= 0)
{
continue;
}
int s = blockCount * levelSize;
System.Diagnostics.Debug.Assert(finalOffset + (ulong)s <= endOffset);
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->State = EntryState.Empty;
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
newEntry->Level = 10;
// link new entry
newEntry->PreviousNear = current;
@@ -200,28 +178,59 @@ namespace StgSharp.HighPerformance.Memory
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// set new entry
// Entry* old = current;
current = newEntry;
// Console.WriteLine($"{(ulong)current->PreviousNear}<->{(ulong)current->NextNear},\t{current->Size}");
// update lock and data
finalOffset += (ulong)s;
remainSize -= (ulong)s;
if (remainSize <= 0)
{
break;
}
}
if (remainSize == 0) {
return;
}
}
finally
{
e->Size = (uint)(offset);
e->Level = GetLevelFromSize((uint)offset);
// Console.WriteLine("Alloc ok");
for (i -= 1; i >= 0; i--)
{
int levelSize = levelSizeArray[i];
ulong levelLong = (ulong)levelSize;
ulong board = endOffset / levelLong * levelLong;
long span = (long)board - (long)finalOffset;
int blockCount = span > 0 ? (int)(span / levelSize) : 0;
if (blockCount <= 0)
{
continue;
}
int s = blockCount * levelSize;
System.Diagnostics.Debug.Assert(finalOffset + (ulong)s <= endOffset);
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->State = EntryState.Empty;
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
// link new entry
newEntry->PreviousNear = current;
newEntry->NextNear = next;
current->NextNear = newEntry;
if (next != null) {
next->PreviousNear = newEntry;
}
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// set new entry
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= (ulong)s;
if (remainSize == 0)
{
break;
}
}
RemoveFromPositionChain(e);
}
/// <summary>
@@ -236,42 +245,49 @@ namespace StgSharp.HighPerformance.Memory
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = _spareMemory->Position;
nuint blockSize = (nuint)levelSizeArray[^1] * 2; // Use last level size (16MB)
nuint bufferEnd = (nuint)m_Buffer + _size;
Entry* previous;
previous = _spareMemory->PreviousNear;
if (_spareMemory->Size <= 0)
if (pos >= bufferEnd || _spareMemory->Size == 0)
{
handle = null;
return false; // No spare memory available
}
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
nuint desired = (nuint)levelSizeArray[^1] * 4;
nuint remain = bufferEnd - pos;
nuint blockSize = desired <= remain ? desired : remain;
if (blockSize == 0)
{
newPos = (nuint)m_Buffer + _size; // Clamp to maximum buffer size
blockSize = _spareMemory->Size;
previous = null;
handle = null;
return false;
}
nuint newPos = pos + blockSize;
// Try to allocate new Entry from slab allocator
handle = (Entry*)_entries.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->Level = GetLevelFromSize((uint)blockSize);
// Update spare memory position
// Update spare memory position & remaining size
_spareMemory->Position = newPos;
_spareMemory->Size = (uint)(bufferEnd - newPos);
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
// update position linked list
Entry* previous = _spareMemory->PreviousNear;
handle->NextNear = _spareMemory;
handle->PreviousNear = previous;
_spareMemory->PreviousNear = handle; // Replace Volatile.Write with direct access
if (previous != null) {
if (previous != null)
{
previous->NextNear = handle;
} else
{
_positionHead = handle;
}
return true;
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+1 -1
View File
@@ -138,7 +138,7 @@ namespace StgSharp.HighPerformance.Memory
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node) {
return TryPopLevel(levelIndex, sizeIndex, out _);
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+49 -4
View File
@@ -39,6 +39,54 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Minimal collect: linear scan, merge contiguous empty Blocks into one, push merged head
/// back to bucket. Caller-responsible design: no slicing or redistribution.
/// </summary>
public void Collect()
{
Entry* cur = _positionHead;
while (cur != null && cur != _spareMemory)
{
if (cur->State != EntryState.Empty)
{
cur = cur->NextNear;
continue;
}
Entry* runHead = cur;
Entry* next = runHead->NextNear;
// Remove head from bucket if currently registered
int headLevel = runHead->Level;
int headSeg = DetermineSegmentIndex(runHead->Size, headLevel);
TryRemoveFromBucket(headLevel, headSeg, (BucketNode*)runHead->Position);
while (next != null &&
next != _spareMemory &&
next->State == EntryState.Empty &&
IsAdjacentTo(runHead, next))
{
int lvl = next->Level;
int seg = DetermineSegmentIndex(next->Size, lvl);
TryRemoveFromBucket(lvl, seg, (BucketNode*)next->Position);
runHead->Size += next->Size;
RemoveFromPositionChain(next);
next = runHead->NextNear;
}
// Recompute level for merged block and push once
runHead->Level = GetLevelFromSize(runHead->Size);
int finalLevel = runHead->Level;
int finalSeg = DetermineSegmentIndex(runHead->Size, finalLevel);
((BucketNode*)runHead->Position)->EntryRef = runHead; // ensure consistency
PushLevel(finalLevel, finalSeg, (BucketNode*)runHead->Position);
cur = next;
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
@@ -55,7 +103,7 @@ namespace StgSharp.HighPerformance.Memory
return;
}
// Set to ThreadOccupied to prevent other operations
// Mark as empty so it can be merged
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
@@ -66,9 +114,6 @@ namespace StgSharp.HighPerformance.Memory
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Set the merged entry state to Empty (ready to be reused)
e->State = EntryState.Empty;
// Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+62 -107
View File
@@ -37,47 +37,13 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
/// <param name="currentOffset">
///
/// </param>
/// <param name="target">
///
/// </param>
/// <param name="level">
///
/// </param>
/// <returns>
///
/// </returns>
private bool AssertBackwardBoundary(ulong currentOffset, Entry* target, int level)
private bool AssertBoundary(Entry* previous, Entry* current, int level)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = levelSizeArray[level] * 4;
ulong targetHead = target->Position - (ulong)m_Buffer;
return (currentOffset / (ulong)nextLevelSize) == (targetHead / (ulong)nextLevelSize);
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
/// <param name="currentOffset">
///
/// </param>
/// <param name="target">
///
/// </param>
/// <param name="level">
///
/// </param>
/// <returns>
///
/// </returns>
private bool AssertForwardBoundary(ulong currentOffset, Entry* target, int level)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = levelSizeArray[level] * 4;
ulong targetTail = target->Position - (ulong)m_Buffer + target->Size - 1;
return (currentOffset / (ulong)nextLevelSize) == (targetTail / (ulong)nextLevelSize);
ulong prevHead = previous->Position - (ulong)m_Buffer,
currHead = current->Position - (ulong)m_Buffer;
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
}
/// <summary>
@@ -108,90 +74,62 @@ namespace StgSharp.HighPerformance.Memory
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
// TODO
/*
* 1.有限回收，最多向上收集2层（算上当前总共3层）
* 2.单层内就可以通过前后链表遍历解决，很容易做到
* 3.由于SLAB分配器内部包含快速复用路径，因而可以考虑直接进行Free掉被合并的Entry。复用成本很低（因为SLAB既没有Version也没有IEnumerable，所以不会有额外的操作计数损耗）
* 4.后续提供HLSF.Collect全量整理API，从头遍历邻接块进行彻底的合并
*/
// TODO 后续提供HLSF.Collect全量整理API，从头遍历邻接块进行彻底的合并
if (entry == null)
{
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(size, originLevel);
// RemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position);
do
{
// previous direction merge
Entry* p = entry->PreviousNear;
if (p == null ||
p->State != EntryState.Empty ||
!IsAdjacentTo(result, p) ||
p->Level != entry->Level ||
!AssertBoundary(p, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
entry->Size += p->Size;
entry->Position = p->Position;
RemoveFromBucket(originLevel, DetermineSegmentIndex(p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
do
{
originLevel = entry->Level;
while (true)
// next direction merge
Entry* n = entry->NextNear;
if (n == null ||
n->State != EntryState.Empty ||
!IsAdjacentTo(result, n) ||
n->Level != entry->Level ||
!AssertBoundary(n, entry, originLevel))
{
// backward merging - simplified to single attempt
Entry* prev = entry->PreviousNear;
// compute offset of current block and base position
ulong currentOffset = entry->Position - (ulong)m_Buffer;
if (prev != null &&
prev != _spareMemory &&
AssertBackwardBoundary(currentOffset, prev, originLevel) &&
IsAdjacentTo(entry, prev) &&
TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
// Merge
entry->Position = prev->Position;
entry->Size += prev->Size;
RemoveFromPositionChain(prev);
}
// Update linked list
/*
Entry* prevPrev = prev->PreviousNear;
if (prevPrev != null) {
prevPrev->NextNear = entry;
}
entry->PreviousNear = prevPrev;
/**/
break;
}
while (true)
{
// forward merging - simplified to single attempt
Entry* next = entry->NextNear;
// Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
entry->Size += n->Size;
RemoveFromBucket(originLevel, DetermineSegmentIndex(n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += entry->Size >= levelSizeArray[originLevel] * 4 ? 1 : 0;
entry->Level &= 15;
ulong currentOffset = entry->Position - (ulong)m_Buffer;
if (next == null ||
next == _spareMemory ||
!AssertForwardBoundary(currentOffset, next, entry->Level) ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
break;
}
// Update linked list
/*
Entry* nextNext = next->NextNear;
if (nextNext != null) {
nextNext->PreviousNear = entry;
}
entry->NextNear = nextNext;
/**/
// Merge
entry->Size += next->Size;
RemoveFromPositionChain(next);
}
entry->Level = GetLevelFromSize(entry->Size);
} while (originLevel < entry->Level);
BucketNode* bucket = (BucketNode*)entry->Position;
bucket->EntryRef = entry;
}
/// <summary>
@@ -202,9 +140,14 @@ namespace StgSharp.HighPerformance.Memory
Entry* prev = entry->PreviousNear;
Entry* next = entry->NextNear;
if (prev != null) {
if (prev != null)
{
prev->NextNear = entry->NextNear;
} else
{
_positionHead = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
@@ -228,11 +171,23 @@ namespace StgSharp.HighPerformance.Memory
[FieldOffset(8)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(28)] internal int _Level; // Level of the block in the allocator
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(32)] public EntryState State;
[FieldOffset(0)] public nuint Position; // Memory block position
public int Level
{
get => _Level;
set
{
if (value > 15 || value < 0) {
throw new InvalidOperationException();
}
_Level = value;
}
}
}
internal enum EntryState : uint
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
+3
View File
@@ -39,6 +39,9 @@ namespace StgSharp.HighPerformance.Memory
// Bucket system field - directly in main allocator class
private BucketNode*[] _bucketHeads;
// Head of the position linked list (first Entry in memory order)
private Entry* _positionHead;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
StgSharpDebug/HlsfStressTest.cs
+8 -2
View File
@@ -42,7 +42,7 @@ namespace StgSharp
int.TryParse(mbStr, out int mb) &&
mb >= 64 ?
(ulong)mb * 1024UL * 1024UL :
1024UL * 1024UL * 1024UL; // lower default to 512MB
3UL * 1024UL * 1024UL * 1024UL; // lower default to 512MB
private static readonly uint[] SizeSweep = {
64, 128, 192, 256, 512, 768, 1024, 2048, 3072, 4096, 8192, 12288, 16384, 32768, 49152, 65536, 131072, 196608, 262144, 524288, 786432, 1_048_576, 2_097_152, 3_145_728, 4_194_304, 8_388_608, 12_582_912, 16_777_216, 25_165_824, 33_554_432
@@ -64,6 +64,7 @@ namespace StgSharp
{
uint size = sz > MaxAllocSize ? MaxAllocSize : sz;
BenchAllocThroughput(allocator, size, touch:false);
allocator.Collect();
GC.Collect();
GC.WaitForPendingFinalizers();
}
@@ -79,6 +80,7 @@ namespace StgSharp
{
uint size = sz > MaxAllocSize ? MaxAllocSize : sz;
BenchAllocThroughput(allocator, size, touch:true);
allocator.Collect();
GC.Collect();
GC.WaitForPendingFinalizers();
}
@@ -90,6 +92,7 @@ namespace StgSharp
Console.WriteLine("[3] Free throughput after pre-allocation (256k blocks of 1KB)");
BenchFreeThroughput(allocator, 256 * 1024, 1024);
allocator.Collect();
}
Console.WriteLine();
Console.WriteLine("[4] Alloc+Immediate Free pairs (5,000,000 x 256B)");
@@ -98,6 +101,7 @@ namespace StgSharp
Warmup(allocator);
BenchAllocFreePairs(allocator, 5_000_000, 256);
allocator.Collect();
}
Console.WriteLine();
Console.WriteLine("[5] Fragmentation cycle (allocate ~1024MB random, free half, reallocate)");
@@ -106,6 +110,7 @@ namespace StgSharp
Warmup(allocator);
BenchFragmentationCycle(allocator, 512); // target within 500MB arena
allocator.Collect();
}
Console.WriteLine();
Console.WriteLine("=== Benchmark Complete ===");
@@ -156,7 +161,8 @@ namespace StgSharp
}
}
list.Add(h);
Console.WriteLine(i);
// Console.WriteLine(i);
}
sw.Stop();
double allocOpsPerSec = ops / sw.Elapsed.TotalSeconds;
StgSharpDebug/Program.cs
+2 -2
View File
@@ -45,9 +45,9 @@ namespace StgSharp
.InitializeModule();
// TestRender();
TestHlsf();
// TestHlsf();
// HlsfStressTest.Run();
HlsfStressTest.Run();
/*
HybridLayerSegregatedFitAllocator hlsf = new HybridLayerSegregatedFitAllocator((nuint)(64 * (ulong)10000 * 10000));