youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reset hlsf to single thread

Browse Source
This commit is contained in:
Youmu Liu
2025-08-23 22:05:47 +08:00
parent 15bd35fbfe
commit 11b630ffa2
6 changed files with 441 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+106 -170
View File
@@ -82,23 +82,13 @@ namespace StgSharp.HighPerformance.Memory
}
}
try
if (handle != null)
{
if (handle != null)
{
AcquireEntryNextLock(handle);
uint allocSize = (uint)((segmentIndex + 1) * levelSizeArray[level]);
SliceMemory(handle, (segmentIndex + 1) * levelSizeArray[level]);
handle->Level = level;
handle->Size = allocSize;
SetEntryState(handle, EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
finally
{
ReleaseEntrySpinLock(handle);
SliceMemory(handle, (segmentIndex + 1) * levelSizeArray[level]);
SetEntryState(handle, EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset)
@@ -112,123 +102,94 @@ namespace StgSharp.HighPerformance.Memory
ulong end = e->Position + e->Size - (ulong)m_Buffer;
int remainSize = (int)(e->Size - offset);
AcquireEntryBothLocks(e);
if (next != null) {
AcquireEntryPrevLock(next);
if ((originOffset & 63) != 0) {
throw new InvalidOperationException("Entry position offset is not aligned to 64 bytes.");
}
try
int minLevel = Math.Max((BitOperations.TrailingZeroCount(originOffset) - 6) / 2, 0);
int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel;
for (; i < levelSizeArray.Length; i++)
{
if ((originOffset & 63) != 0) {
throw new InvalidOperationException("Entry position offset is not aligned to 64 bytes.");
}
int minLevel = Math.Max((BitOperations.TrailingZeroCount(originOffset) - 6) / 2, 0);
int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel;
for (; i < levelSizeArray.Length; i++)
int levelSize = levelSizeArray[i];
ulong board = (finalOffset + (ulong)levelSize - 1) & ~((ulong)levelSize - 1);
int blockCount = (int)(ulong.Min(board, end) / (ulong)levelSize);
if (blockCount == 0)
{
int levelSize = levelSizeArray[i];
ulong board = (finalOffset + (ulong)levelSize - 1) & ~((ulong)levelSize - 1);
int blockCount = (int)(ulong.Min(board, end) / (ulong)levelSize);
if (blockCount == 0)
{
break;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
newEntry->LockBuffer = 0;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
bucket->LockBuffer = 0;
newEntry->State = EntryState.ThreadOccupied;
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
if (next != null) {
next->PreviousNear = (ulong)newEntry;
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
Entry* old = current;
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
ReleaseEntryNextLock(old);
AcquireEntryNextLock(current);
break;
}
int s = blockCount * levelSize;
if (remainSize <= 0) {
return;
}
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
newEntry->State = EntryState.ThreadOccupied;
for (; i >= 0; i--)
{
int levelSize = levelSizeArray[i];
ulong board = (finalOffset + (ulong)levelSize - 1) & ~((ulong)levelSize - 1);
int blockCount = (int)(ulong.Min(board, end) / (ulong)levelSize);
if (blockCount == 0)
{
continue;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->LockBuffer = 0;
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
bucket->LockBuffer = 0;
newEntry->State = EntryState.ThreadOccupied;
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
if (next != null) {
next->PreviousNear = (ulong)newEntry;
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
Entry* old = current;
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
ReleaseEntryNextLock(old);
AcquireEntryNextLock(current);
if (remainSize <= 0)
{
break;
}
}
}
catch (Exception)
{
throw;
}
finally
{
ReleaseEntryBothLocks(e);
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
if (next != null) {
ReleaseEntryPrevLock(next);
next->PreviousNear = (ulong)newEntry;
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
current = newEntry;
// update data
finalOffset += (ulong)s;
remainSize -= s;
}
if (remainSize <= 0) {
return;
}
for (i -= 1; i >= 0; i--)
{
int levelSize = levelSizeArray[i];
ulong board = (finalOffset + (ulong)levelSize - 1) & ~((ulong)levelSize - 1);
int blockCount = (int)(ulong.Min(board, end) / (ulong)levelSize);
if (blockCount == 0)
{
continue;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
newEntry->State = EntryState.ThreadOccupied;
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
if (next != null) {
next->PreviousNear = (ulong)newEntry;
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
current = newEntry;
// update data
finalOffset += (ulong)s;
remainSize -= s;
if (remainSize <= 0)
{
break;
}
}
}
@@ -244,22 +205,13 @@ namespace StgSharp.HighPerformance.Memory
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint pos = _spareMemory->Position;
nuint blockSize = (nuint)levelSizeArray[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
ulong p;
Entry* previous;
do
{
p = Volatile.Read(ref _spareMemory->PreviousNear);
previous = (Entry*)p;
if (previous != null) {
AcquireEntryNextLock(previous);
}
AcquireEntryPrevLock(_spareMemory);
break;
} while (true);
ulong p = _spareMemory->PreviousNear;
Entry* previous = (Entry*)p;
if (_spareMemory->Size <= 0)
{
handle = null;
@@ -274,42 +226,26 @@ namespace StgSharp.HighPerformance.Memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_entries.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
SetEntryState(handle, EntryState.ThreadOccupied);
handle->Level = GetLevelFromSize((uint)blockSize);
handle = (Entry*)_entries.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
SetEntryState(handle, EntryState.ThreadOccupied);
handle->Level = GetLevelFromSize((uint)blockSize);
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
// Update spare memory position
_spareMemory->Position = newPos;
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
// update position linked list
handle->NextNear = (ulong)_spareMemory;
handle->PreviousNear = p;
_spareMemory->PreviousNear = (ulong)handle;
if (previous != null) {
previous->NextNear = (ulong)handle;
}
return true;
}
catch
{
handle = null;
return false;
}
finally
{
ReleaseEntryNextLock(previous);
ReleaseEntryPrevLock(_spareMemory);
// update position linked list
handle->NextNear = (ulong)_spareMemory;
handle->PreviousNear = p;
_spareMemory->PreviousNear = (ulong)handle;
if (previous != null) {
previous->NextNear = (ulong)handle;
}
return true;
}
/// <summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+84 -407
View File
@@ -39,12 +39,6 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
@@ -53,6 +47,16 @@ namespace StgSharp.HighPerformance.Memory
_bucketHeads = new ulong[levelSizeArray.Length * 3];
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
#endregion
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
@@ -60,270 +64,112 @@ namespace StgSharp.HighPerformance.Memory
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
if (_bucketHeads[index] == EmptyHandle)
{
_bucketHeads[index] = (nuint)node;
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
} else
{
while (true)
{
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
/// <summary>
/// 根据锁类型释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
{
switch (lockType)
{
case 0:
ReleaseBucketNodePrevLock(node);
return; // PrevLock
case 1:
ReleaseBucketNodeBothLocks(node);
return; // BothLocks
case 2:
ReleaseBucketNodeNextLock(node);
return; // NextLock
default:
return;
}
}
// Add to head of linked list
ulong head = _bucketHeads[index];
BucketNode* headNode = (BucketNode*)head;
/// <summary>
/// 按地址顺序释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
if (previous != null) {
ReleaseBucketNodeNextLock(previous);
}
if (node != null) {
ReleaseBucketNodeBothLocks(node);
}
if (next != null) {
ReleaseBucketNodePrevLock(next);
node->NextLevel = head;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
_bucketHeads[index] = (nuint)node;
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
if ((BucketNode*)_bucketHeads[index] == node)
{
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
Thread.Sleep(0);
continue;
}
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
if (!TryAcquireOrderedBucketLocks(previous, node, next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
{
Thread.Sleep(0);
continue;
}
if (previous != null) {
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
_bucketHeads[index] = EmptyHandle;
return;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
return TryAcquireBucketNodeBothLocks(node);
}
ulong p = node->PreviousLevel;
ulong n = node->NextLevel;
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
if (previous != null && node != null && next != null)
{
// previous -> node -> next
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next);
} else if (previous != null && node != null) {
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
if (previous != null) {
previous->NextLevel = n;
}
return true;
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
ulong h = _bucketHeads[index];
BucketNode* head = (BucketNode*)h;
if (head == null)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
if (next == null)
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], EmptyHandle, h) == h)
{
node = head;
return true;
}
node = null;
return false;
}
try
{
if (!TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
node = null;
return false;
}
ulong n = head->NextLevel;
BucketNode* next = (BucketNode*)n;
if (next == null)
{
_bucketHeads[index] = EmptyHandle;
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return true;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
if ((BucketNode*)_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return true;
}
return false;
}
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
if (!TryAcquireOrderedBucketLocks(previous, node, next)) {
return false;
_bucketHeads[index] = EmptyHandle;
return true;
}
try
{
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n) {
return false;
}
ulong p = node->PreviousLevel;
ulong n = node->NextLevel;
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
if (previous != null) {
previous->NextLevel = n;
@@ -337,9 +183,9 @@ namespace StgSharp.HighPerformance.Memory
node->IsInBucket = false;
return true;
}
finally
catch
{
ReleaseOrderedBucketLocks(previous, node, next);
return false;
}
}
@@ -347,7 +193,7 @@ namespace StgSharp.HighPerformance.Memory
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct BucketNode
{
@@ -355,181 +201,12 @@ namespace StgSharp.HighPerformance.Memory
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock;
[FieldOffset(32)] internal int PrevLock;
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(32)] internal ulong LockBuffer;
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作（控制PreviousLevel字段）
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref node->PrevLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作（控制NextLevel字段）
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0 ||
Volatile.Read(ref node->NextLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
int retryCount = 0;
while (retryCount < 1000)
{
if (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref node->PrevLock) == thread)
{
if (Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0 ||
Volatile.Read(ref node->NextLock) == thread)
{
return true;
} else
{
Volatile.Write(ref node->PrevLock, 0);
}
}
retryCount++;
Thread.Sleep(0); // 让出 CPU
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
while (true)
{
if (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref node->PrevLock) == thread)
{
if (Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0 ||
Volatile.Read(ref node->NextLock) == thread)
{
return;
} else
{
Volatile.Write(ref node->PrevLock, 0);
}
}
Thread.Sleep(0); // 让出 CPU
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
#endregion
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+24 -25
View File
@@ -47,33 +47,32 @@ namespace StgSharp.HighPerformance.Memory
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (GetEntryState(e) != EntryState.Allocated) {
return;
}
SetEntryState(e, EntryState.ThreadOccupied);
// Step 4: Try merge with adjacent free blocks and remove them from buckets
MergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
SetEntryState(e, EntryState.Empty);
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
if (GetEntryState(e) != EntryState.Allocated) {
return;
}
// Set to ThreadOccupied to prevent other operations
SetEntryState(e, EntryState.ThreadOccupied);
// Try merge with adjacent free blocks and remove them from buckets
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
uint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Set the merged entry state to Empty (ready to be reused)
SetEntryState(e, EntryState.Empty);
// Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+90 -296
View File
@@ -41,12 +41,6 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntrySpinLock(Entry* entry)
{
AcquireEntryBothLocks(entry);
}
private bool AssertBackwardBoundary(ulong current, Entry* target, int level)
{
ulong align = (ulong)levelSizeArray[level + 1];
@@ -83,8 +77,7 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy, any fail in acquiring spin lock will end a merging process.
/// Current Entry will serve as the final merged block.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
@@ -92,185 +85,127 @@ namespace StgSharp.HighPerformance.Memory
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry)) {
return;
}
if (!TryRemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position)) { }
try
RemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position);
SetEntryState(entry, EntryState.Merging);
while (originLevel >= entry->Level)
{
_ = TrySetEntryState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
originLevel = entry->Level;
while (true)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null || // null ref
prev == _spareMemory || // point at end of list
GetEntryState(prev) != EntryState.Empty || // not empty
!IsAdjacentTo(entry, prev) || // not nearby
entry->Level != prev->Level || // not same level
!AssertBackwardBoundary(offset, prev, entry->Level) || // not aligned if merge
!TryAcquireEntrySpinLock(prev)) // occupied by threads
{
break;
}
ulong pp = prev->NextNear;
Entry* prevPrev = (Entry*)pp;
if (!TryAcquireEntryNextLock(prevPrev))
{
break;
}
try
{
if (!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
break;
}
entry->Position = prev->Position; // Update position to the previous entry
entry->Size += prev->Size; // Merge sizes
prevPrev->NextNear = (ulong)entry;
RecycleEntry(prev);
}
finally
{
ReleaseEntryNextLock(prevPrev);
ReleaseEntrySpinLock(prev);
}
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null || // null ref
next == _spareMemory || // point at end of list
GetEntryState(next) != EntryState.Empty || // not empty
!IsAdjacentTo(entry, next) || // not nearby
entry->Level != next->Level || // not same level
!AssertBackwardBoundary(offset, next, entry->Level) || // not aligned if merge
!TryAcquireEntrySpinLock(next)) // occupied by threads
{
break;
}
ulong nn = next->NextNear;
Entry* nextNext = (Entry*)nn;
if (!TryAcquireEntryPrevLock(nextNext))
{
break;
}
try
{
if (!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
break;
}
entry->Size += next->Size;
entry->NextNear = next->NextNear;
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
entry->Level = GetLevelFromSize(entry->Size);
if (originLevel == entry->Level)
// backward merging - simplified to single attempt
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev != null ||
prev != _spareMemory ||
GetEntryState(prev) == EntryState.Empty ||
!AssertBackwardBoundary(entry->Position, prev, originLevel) ||
IsAdjacentTo(entry, prev) ||
entry->Level == prev->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
break;
}
originLevel = entry->Level;
// Update linked list
Entry* prevPrev = (Entry*)prev->PreviousNear;
if (prevPrev != null) {
prevPrev->NextNear = (ulong)entry;
}
entry->PreviousNear = (ulong)prevPrev;
// Merge
entry->Position = prev->Position;
entry->Size += prev->Size;
RecycleEntry(prev);
}
_ = TrySetEntryState(entry, EntryState.Merging, EntryState.Empty);
return;
}
finally
{
ReleaseEntrySpinLock(entry);
while (true)
{
// forward merging - simplified to single attempt
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next != null ||
next != _spareMemory ||
GetEntryState(next) == EntryState.Empty ||
!AssertForwardBoundary(entry->Position + entry->Size, next, entry->Level) ||
IsAdjacentTo(entry, next) ||
entry->Level == next->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
break;
}
// Update linked list
Entry* nextNext = (Entry*)next->NextNear;
if (nextNext != null) {
nextNext->PreviousNear = (ulong)entry;
}
entry->NextNear = (ulong)nextNext;
// Merge
entry->Size += next->Size;
RecycleEntry(next);
}
entry->Level = GetLevelFromSize(entry->Size);
}
SetEntryState(entry, EntryState.Empty);
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
if (entry == null) {
return;
}
AcquireEntrySpinLock(entry);
entry->LockBuffer = 0;
entry->State = EntryState.Empty;
entry->PreviousNear = EmptyHandle;
entry->NextNear = EmptyHandle;
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntrySpinLock(Entry* entry)
{
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
if (entry == null) {
return;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
return TryAcquireEntryBothLocks(entry);
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal int NextLock;
[FieldOffset(36)] internal int PrevLock;
[FieldOffset(40)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(20)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(32)] internal ulong LockBuffer;
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
@@ -306,19 +241,26 @@ namespace StgSharp.HighPerformance.Memory
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
if (entry != null && entry->State == expectedState)
{
entry->State = newState;
return true;
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry)
{
return Volatile.Read(ref entry->State);
return entry != null ? entry->State : 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
Volatile.Write(ref entry->State, newState);
if (entry != null) {
entry->State = newState;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -345,154 +287,6 @@ namespace StgSharp.HighPerformance.Memory
return GetEntryState(entry) == EntryState.Merging;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
// 尝试同时获取前后锁
if (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread)
{
if (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread)
{
return true;
} else
{
// 回滚前锁
Volatile.Write(ref entry->PrevLock, 0);
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
// 循环直到成功获取两个锁
while (true)
{
if ((Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread) &&
(Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread)) {
return;
}
// 回滚前锁
Volatile.Write(ref entry->PrevLock, 0);
Thread.Sleep(0);
}
}
#endregion
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
-4
View File
@@ -66,8 +66,6 @@ namespace StgSharp.HighPerformance.Memory
_spareMemory->PreviousNear = EmptyHandle;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
_spareMemory->Size = (uint)byteSize;
}
@@ -89,8 +87,6 @@ namespace StgSharp.HighPerformance.Memory
_spareMemory->PreviousNear = EmptyHandle;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
_spareMemory->Size = (uint)byteSize;
}
StgSharpDebug/Program.cs
+137 -36
View File
@@ -63,62 +63,163 @@ namespace StgSharp
// HLSF test arena
Console.WriteLine(
"=== HLSF (Hybrid Layer Segregated Fit Allocator) Multi-threaded Test Arena ===\n");
HybridLayerSegregatedFitAllocator hlsfAllocator = new HybridLayerSegregatedFitAllocator(1024 * 1024 * 4);
"=== HLSF (Hybrid Layer Segregated Fit Allocator) Single-threaded Stress Test ===\n");
int threadCount = 4;
int blocksPerThread = 1024;
int blockSize = 1024;
Thread[] threads = new Thread[threadCount];
for (int t = 0; t < threadCount; t++)
// 使用适中的内存池大小
HybridLayerSegregatedFitAllocator hlsfAllocator = new HybridLayerSegregatedFitAllocator(16 * 1024 * 1024);
Console.WriteLine("开始单线程压力测试...");
// 单线程压力测试：适度的分配和释放操作
int totalCycles = 5;
int allocationsPerCycle = 1024 * 64; // 65536 次分配
for (int cycle = 0; cycle < totalCycles; cycle++)
{
threads[t] = new Thread(() => {
int threadIndex = Environment.CurrentManagedThreadId;
Stack<HybridLayerSegregatedFitAllocationHandle> handles = new();
Console.WriteLine($"\n=== 第 {cycle + 1} 轮压力测试 ===");
// Allocate blocks
for (int i = 0; i < blocksPerThread; i++)
// 测试不同大小的分配
List<HybridLayerSegregatedFitAllocationHandle> handles = new List<HybridLayerSegregatedFitAllocationHandle>();
Random random = new Random(cycle); // 使用固定种子以确保可重现性
// 分配阶段
Console.WriteLine($"分配阶段：分配 {allocationsPerCycle} 个内存块...");
Stopwatch stopwatch = System.Diagnostics.Stopwatch.StartNew();
for (int i = 0; i < allocationsPerCycle; i++)
{
// 随机大小：64字节到1KB
uint size = (uint)(64 + random.Next(1024 - 64 + 1));
handles.Add(hlsfAllocator.Alloc(size));
if (i % 5000 == 0 && i > 0) {
Console.WriteLine($" 已分配 {i} 个块");
}
}
stopwatch.Stop();
Console.WriteLine($"分配完成，总计分配 {handles.Count} 个块");
Console.WriteLine($"分配耗时: {stopwatch.ElapsedMilliseconds} ms");
if (stopwatch.ElapsedMilliseconds > 0) {
Console.WriteLine($"分配速度: {allocationsPerCycle * 1000 / stopwatch.ElapsedMilliseconds} allocs/sec");
}
// 随机释放阶段
Console.WriteLine("随机释放阶段...");
stopwatch.Restart();
// 随机打乱释放顺序
for (int i = handles.Count - 1; i > 0; i--)
{
int j = random.Next(i + 1);
(handles[i], handles[j]) = (handles[j], handles[i]);
}
for (int i = 0; i < handles.Count; i++)
{
hlsfAllocator.Free(handles[i]);
if (i % 5000 == 0 && i > 0) {
Console.WriteLine($" 已释放 {i} 个块");
}
}
stopwatch.Stop();
Console.WriteLine($"释放完成，总计释放 {handles.Count} 个块");
Console.WriteLine($"释放耗时: {stopwatch.ElapsedMilliseconds} ms");
if (stopwatch.ElapsedMilliseconds > 0) {
Console.WriteLine($"释放速度: {handles.Count * 1000 / stopwatch.ElapsedMilliseconds} frees/sec");
}
handles.Clear();
// 碎片化测试
Console.WriteLine("碎片化测试：交替分配释放...");
List<HybridLayerSegregatedFitAllocationHandle> fragmentationHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
for (int i = 0; i < 500; i++) // 减少碎片化测试次数
{
// 分配5个块
for (int j = 0; j < 5; j++)
{
Console.WriteLine($"Thread {threadIndex}: Allocating block {i}.");
handles.Push(hlsfAllocator.Alloc((uint)blockSize));
uint size = (uint)(64 << (i % 4)); // 64, 128, 256, 512
fragmentationHandles.Add(hlsfAllocator.Alloc(size));
}
// Free blocks
while (handles.TryPop(out HybridLayerSegregatedFitAllocationHandle handle))
// 释放前3个块，保留后2个
for (int j = 0; j < 3 && fragmentationHandles.Count > 0; j++)
{
Console.WriteLine($"Thread {threadIndex}: Freeing a block.");
hlsfAllocator.Free(handle);
hlsfAllocator.Free(fragmentationHandles[0]);
fragmentationHandles.RemoveAt(0);
}
}
Console.WriteLine($"Thread {threadIndex}: All blocks allocated and freed.");
});
// 释放剩余的块
foreach (HybridLayerSegregatedFitAllocationHandle handle in fragmentationHandles)
{
hlsfAllocator.Free(handle);
}
Console.WriteLine($"第 {cycle + 1} 轮测试完成");
}
// Start all threads
foreach (Thread thread in threads)
Console.WriteLine("\n=== 边界测试 ===");
// 测试各种边界情况
Console.WriteLine("测试最小分配 (64字节)...");
List<HybridLayerSegregatedFitAllocationHandle> minHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
for (int i = 0; i < 1000; i++) {
minHandles.Add(hlsfAllocator.Alloc(64));
}
foreach (HybridLayerSegregatedFitAllocationHandle handle in minHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("最小分配测试完成");
Console.WriteLine("测试最大分配 (1024字节)...");
List<HybridLayerSegregatedFitAllocationHandle> maxHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
for (int i = 0; i < 1000; i++) {
maxHandles.Add(hlsfAllocator.Alloc(1024));
}
foreach (HybridLayerSegregatedFitAllocationHandle handle in maxHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("最大分配测试完成");
Console.WriteLine("测试混合大小分配...");
List<HybridLayerSegregatedFitAllocationHandle> mixedHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
uint[] testSizes = {
64, 128, 256, 512, 1024
};
for (int i = 0; i < 2000; i++)
{
thread.Start();
uint size = testSizes[i % testSizes.Length];
mixedHandles.Add(hlsfAllocator.Alloc(size));
}
// Wait for all threads to complete
foreach (Thread thread in threads)
{
thread.Join();
foreach (HybridLayerSegregatedFitAllocationHandle handle in mixedHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("混合大小分配测试完成");
Console.WriteLine("All threads completed successfully.");
Console.WriteLine("\n所有测试完成！HLSF分配器表现良好。");
Console.WriteLine("按任意键退出...");
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex);
/*
World.WriteLog($"{ex}", LogType.Error);
if( true ) { }
Dialogue.PostError( ex );
/**/
Console.WriteLine($"测试过程中发生错误: {ex}");
Console.WriteLine("按任意键退出...");
Console.ReadKey();
World.Terminate(-1);
/**/
}
finally
{