youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reset hlsf to single thread

Browse Source
This commit is contained in:
Youmu Liu
2025-08-23 22:05:47 +08:00
parent 15bd35fbfe
commit 11b630ffa2
6 changed files with 441 additions and 938 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+90 -296
View File
@@ -41,12 +41,6 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntrySpinLock(Entry* entry)
{
AcquireEntryBothLocks(entry);
}
private bool AssertBackwardBoundary(ulong current, Entry* target, int level)
{
ulong align = (ulong)levelSizeArray[level + 1];
@@ -83,8 +77,7 @@ namespace StgSharp.HighPerformance.Memory
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy, any fail in acquiring spin lock will end a merging process.
/// Current Entry will serve as the final merged block.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
@@ -92,185 +85,127 @@ namespace StgSharp.HighPerformance.Memory
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry)) {
return;
}
if (!TryRemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position)) { }
try
RemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position);
SetEntryState(entry, EntryState.Merging);
while (originLevel >= entry->Level)
{
_ = TrySetEntryState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
originLevel = entry->Level;
while (true)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null || // null ref
prev == _spareMemory || // point at end of list
GetEntryState(prev) != EntryState.Empty || // not empty
!IsAdjacentTo(entry, prev) || // not nearby
entry->Level != prev->Level || // not same level
!AssertBackwardBoundary(offset, prev, entry->Level) || // not aligned if merge
!TryAcquireEntrySpinLock(prev)) // occupied by threads
{
break;
}
ulong pp = prev->NextNear;
Entry* prevPrev = (Entry*)pp;
if (!TryAcquireEntryNextLock(prevPrev))
{
break;
}
try
{
if (!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
break;
}
entry->Position = prev->Position; // Update position to the previous entry
entry->Size += prev->Size; // Merge sizes
prevPrev->NextNear = (ulong)entry;
RecycleEntry(prev);
}
finally
{
ReleaseEntryNextLock(prevPrev);
ReleaseEntrySpinLock(prev);
}
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null || // null ref
next == _spareMemory || // point at end of list
GetEntryState(next) != EntryState.Empty || // not empty
!IsAdjacentTo(entry, next) || // not nearby
entry->Level != next->Level || // not same level
!AssertBackwardBoundary(offset, next, entry->Level) || // not aligned if merge
!TryAcquireEntrySpinLock(next)) // occupied by threads
{
break;
}
ulong nn = next->NextNear;
Entry* nextNext = (Entry*)nn;
if (!TryAcquireEntryPrevLock(nextNext))
{
break;
}
try
{
if (!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
break;
}
entry->Size += next->Size;
entry->NextNear = next->NextNear;
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
entry->Level = GetLevelFromSize(entry->Size);
if (originLevel == entry->Level)
// backward merging - simplified to single attempt
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev != null ||
prev != _spareMemory ||
GetEntryState(prev) == EntryState.Empty ||
!AssertBackwardBoundary(entry->Position, prev, originLevel) ||
IsAdjacentTo(entry, prev) ||
entry->Level == prev->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
break;
}
originLevel = entry->Level;
// Update linked list
Entry* prevPrev = (Entry*)prev->PreviousNear;
if (prevPrev != null) {
prevPrev->NextNear = (ulong)entry;
}
entry->PreviousNear = (ulong)prevPrev;
// Merge
entry->Position = prev->Position;
entry->Size += prev->Size;
RecycleEntry(prev);
}
_ = TrySetEntryState(entry, EntryState.Merging, EntryState.Empty);
return;
}
finally
{
ReleaseEntrySpinLock(entry);
while (true)
{
// forward merging - simplified to single attempt
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next != null ||
next != _spareMemory ||
GetEntryState(next) == EntryState.Empty ||
!AssertForwardBoundary(entry->Position + entry->Size, next, entry->Level) ||
IsAdjacentTo(entry, next) ||
entry->Level == next->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
break;
}
// Update linked list
Entry* nextNext = (Entry*)next->NextNear;
if (nextNext != null) {
nextNext->PreviousNear = (ulong)entry;
}
entry->NextNear = (ulong)nextNext;
// Merge
entry->Size += next->Size;
RecycleEntry(next);
}
entry->Level = GetLevelFromSize(entry->Size);
}
SetEntryState(entry, EntryState.Empty);
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
if (entry == null) {
return;
}
AcquireEntrySpinLock(entry);
entry->LockBuffer = 0;
entry->State = EntryState.Empty;
entry->PreviousNear = EmptyHandle;
entry->NextNear = EmptyHandle;
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntrySpinLock(Entry* entry)
{
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
if (entry == null) {
return;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
return TryAcquireEntryBothLocks(entry);
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal int NextLock;
[FieldOffset(36)] internal int PrevLock;
[FieldOffset(40)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(20)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(32)] internal ulong LockBuffer;
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
@@ -306,19 +241,26 @@ namespace StgSharp.HighPerformance.Memory
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
if (entry != null && entry->State == expectedState)
{
entry->State = newState;
return true;
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry)
{
return Volatile.Read(ref entry->State);
return entry != null ? entry->State : 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
Volatile.Write(ref entry->State, newState);
if (entry != null) {
entry->State = newState;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -345,154 +287,6 @@ namespace StgSharp.HighPerformance.Memory
return GetEntryState(entry) == EntryState.Merging;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return
Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
// 尝试同时获取前后锁
if (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread)
{
if (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread)
{
return true;
} else
{
// 回滚前锁
Volatile.Write(ref entry->PrevLock, 0);
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
// 循环直到成功获取两个锁
while (true)
{
if ((Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0 ||
Volatile.Read(ref entry->PrevLock) == thread) &&
(Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0 ||
Volatile.Read(ref entry->NextLock) == thread)) {
return;
}
// 回滚前锁
Volatile.Write(ref entry->PrevLock, 0);
Thread.Sleep(0);
}
}
#endregion
}
}