youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HLSF full functionally available. Working on Matrix.

Browse Source 
Signed-off-by: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com>
This commit is contained in:
Youmu Liu
2025-08-25 22:13:45 +08:00
parent fe097fd94c
commit e5e4b1dd36
1479 changed files with 2083 additions and 619964 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/BufferStack.cs
+5 -3
View File
@@ -87,8 +87,10 @@ namespace StgSharp.HighPerformance.Memory
public void Push(T item)
{
Header* header = (Header*)_buffer;
if (header->Top >= header->Capacity) {
if (header->Top >= header->Capacity)
{
Expand();
header = (Header*)_buffer;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
@@ -125,8 +127,8 @@ namespace StgSharp.HighPerformance.Memory
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart,
header->Capacity * (nuint)sizeof(T),
Buffer.MemoryCopy(sourcePtr, dataStart,
header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+54 -144
View File
@@ -48,8 +48,6 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSizeArray[^1] * 2);
@@ -57,13 +55,7 @@ namespace StgSharp.HighPerformance.Memory
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle)) {
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
Entry* handle;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
@@ -73,25 +65,19 @@ namespace StgSharp.HighPerformance.Memory
}
if (i >= levelSizeArray.Length)
{
if (!TryAllocateNew16MBBlock(out handle)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
}
if (handle != null) {
SliceMemory(handle, (segmentIndex + 1) * levelSizeArray[level]);
}
return new hlsfHandle(handle, size);
}
finally
{
ReleaseEntryNextLock(handle);
}
handle->State = EntryState.Allocated;
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset)
@@ -102,12 +88,10 @@ namespace StgSharp.HighPerformance.Memory
ulong originOffset = e->Position - (ulong)m_Buffer;
ulong finalOffset = originOffset + (ulong)offset;
ulong end = e->Position + e->Size;
int remainSize = (int)(e->Size - offset);
AcquireEntryNextLock(e);
AcquireEntryPrevLock(next);
int minLevel = Math.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0);
int minLevel = int.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0);
int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel;
for (; i < levelSizeArray.Length; i++)
@@ -117,13 +101,9 @@ namespace StgSharp.HighPerformance.Memory
int blockCount = (int)((ulong.Min(board, end) - board) / (ulong)levelSize);
if (blockCount == 0)
{
int levelSize = levelSizeArray[i];
int blockCount = remainSize / levelSize % 4;
if (blockCount == 0)
{
break;
}
int s = blockCount * levelSize;
break;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
@@ -132,25 +112,23 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
next->PreviousNear = (ulong)newEntry;
// link new entry
newEntry->PreviousNear = (ulong)current;
newEntry->NextNear = n;
current->NextNear = (ulong)newEntry;
next->PreviousNear = (ulong)newEntry;
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
Entry* old = current;
current = newEntry;
// set new entry
Entry* old = current;
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
ReleaseEntryNextLock(old);
AcquireEntryNextLock(current);
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
@@ -193,6 +171,18 @@ namespace StgSharp.HighPerformance.Memory
next->PreviousNear = (ulong)newEntry;
}
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
Entry* old = current;
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
@@ -202,65 +192,6 @@ namespace StgSharp.HighPerformance.Memory
// update data
finalOffset += (ulong)s;
remainSize -= s;
if (remainSize <= 0)
{
ReleaseEntryNextLock(current);
ReleaseEntryPrevLock(next);
return;
}
for (; i >= 0; i--)
{
int levelSize = levelSizeArray[i];
int blockCount = remainSize / levelSize % 4;
if (blockCount == 0)
{
continue;
}
int s = blockCount * levelSize;
// create and init new entry
Entry* newEntry = (Entry*)_entries.Allocate();
newEntry->LockBuffer = 0;
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
newEntry->Size = (uint)s;
BucketNode* bucket = (BucketNode*)newEntry->Position;
bucket->EntryRef = newEntry;
bucket->LockBuffer = 0;
newEntry->State = EntryState.ThreadOccupied;
// link new entry
newEntry->PreviousNear = (ulong)current;
current->NextNear = (ulong)newEntry;
newEntry->NextNear = n;
next->PreviousNear = (ulong)newEntry;
newEntry->Level = i;
PushLevel(i, blockCount - 1, bucket);
// set new entry
Entry* old = current;
current = newEntry;
// update lock and data
finalOffset += (ulong)s;
remainSize -= s;
ReleaseEntryNextLock(old);
AcquireEntryNextLock(current);
if (remainSize <= 0)
{
break;
}
}
}
catch (Exception)
{
throw;
}
finally
{
// Release locks in reverse order
ReleaseEntryNextLock(current);
ReleaseEntryPrevLock(next);
}
}
@@ -275,22 +206,14 @@ namespace StgSharp.HighPerformance.Memory
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint pos = _spareMemory->Position;
nuint blockSize = (nuint)levelSizeArray[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
ulong p;
Entry* previous;
do
{
p = Volatile.Read(ref _spareMemory->PreviousNear);
previous = (Entry*)p;
if (previous != null) {
AcquireEntryNextLock(previous);
}
AcquireEntryPrevLock(_spareMemory);
break;
} while (true);
p = _spareMemory->PreviousNear;
previous = (Entry*)p;
if (_spareMemory->Size <= 0)
{
handle = null;
@@ -310,34 +233,21 @@ namespace StgSharp.HighPerformance.Memory
handle->Size = (uint)blockSize;
handle->Level = GetLevelFromSize((uint)blockSize);
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos; // Replace Volatile.Write with direct access
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
// Update spare memory position
_spareMemory->Position = newPos;
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
// update position linked list
handle->NextNear = (ulong)_spareMemory;
handle->PreviousNear = p;
_spareMemory->PreviousNear = (ulong)handle; // Replace Volatile.Write with direct access
if (previous != null) {
previous->NextNear = (ulong)handle;
}
// update position linked list
handle->NextNear = (ulong)_spareMemory;
handle->PreviousNear = p;
_spareMemory->PreviousNear = (ulong)handle;
if (previous != null) {
previous->NextNear = (ulong)handle;
}
return true;
}
catch
{
handle = null;
return false;
}
finally
{
ReleaseEntryNextLock(previous);
ReleaseEntryPrevLock(_spareMemory);
}
return true;
}
/// <summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+75 -381
View File
@@ -39,12 +39,6 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
@@ -53,6 +47,16 @@ namespace StgSharp.HighPerformance.Memory
_bucketHeads = new ulong[levelSizeArray.Length * 3];
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
#endregion
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
@@ -60,286 +64,110 @@ namespace StgSharp.HighPerformance.Memory
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
if (_bucketHeads[index] == EmptyHandle)
{
_bucketHeads[index] = (nuint)node;
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
ulong head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
}
}
/// <summary>
/// 根据锁类型释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
{
switch (lockType)
{
case 0:
ReleaseBucketNodePrevLock(node);
return; // PrevLock
case 1:
ReleaseBucketNodeBothLocks(node);
return; // BothLocks
case 2:
ReleaseBucketNodeNextLock(node);
return; // NextLock
default:
return;
}
}
/// <summary>
/// 按地址顺序释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
if (previous != null) {
ReleaseBucketNodeNextLock(previous);
}
if (node != null) {
ReleaseBucketNodeBothLocks(node);
}
if (next != null) {
ReleaseBucketNodePrevLock(next);
return;
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
if ((BucketNode*)_bucketHeads[index] == node)
{
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
Thread.Sleep(0);
continue;
}
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
if (!TryAcquireOrderedBucketLocks(previous, node, next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
{
Thread.Sleep(0);
continue;
}
if (previous != null) {
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
return TryAcquireBucketNodeBothLocks(node);
}
ulong p = node->PreviousLevel;
ulong n = node->NextLevel;
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
if (previous != null && node != null && next != null)
{
// previous -> node -> next
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next);
} else if (previous != null && node != null) {
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
if (previous != null) {
previous->NextLevel = n;
}
return true;
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
ulong h = _bucketHeads[index];
BucketNode* head = (BucketNode*)h;
if (head == null)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
if (next == null)
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], EmptyHandle, h) == h)
{
node = head;
return true;
}
node = null;
return false;
}
try
{
if (!TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
node = null;
return false;
}
ulong n = head->NextLevel;
BucketNode* next = (BucketNode*)n;
if (next == null)
{
_bucketHeads[index] = EmptyHandle;
node = head;
return true;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return true;
}
return false;
if ((BucketNode*) _bucketHeads[index] == node) {
return TryPopLevel(levelIndex, sizeIndex, out _);
}
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
ulong p = node->PreviousLevel;
ulong n = node->NextLevel;
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
if (!TryAcquireOrderedBucketLocks(previous, node, next)) {
return false;
if (previous != null) {
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
try
{
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n) {
return false;
}
if (previous != null) {
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return true;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return true;
}
/// <summary>
@@ -364,139 +192,5 @@ namespace StgSharp.HighPerformance.Memory
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作（控制PreviousLevel字段）
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作（控制NextLevel字段）
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
-13
View File
@@ -54,37 +54,24 @@ namespace StgSharp.HighPerformance.Memory
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (GetEntryState(e) != EntryState.Allocated) {
return;
}
SetEntryState(e, EntryState.ThreadOccupied);
if (e->State != EntryState.Allocated) {
return;
}
// Set to ThreadOccupied to prevent other operations
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
uint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Set the merged entry state to Empty (ready to be reused)
e->State = EntryState.Empty;
// Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+33 -51
View File
@@ -41,24 +41,18 @@ namespace StgSharp.HighPerformance.Memory
public unsafe partial class HybridLayerSegregatedFitAllocator
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntrySpinLock(Entry* entry)
private bool AssertBackwardBoundary(ulong currentOffset, Entry* target, int level)
{
AcquireEntryBothLocks(entry);
int nextLevelSize = levelSizeArray[level] * 4;
ulong targetHead = target->Position - (ulong)m_Buffer;
return (currentOffset / (ulong)nextLevelSize) == (targetHead / (ulong)nextLevelSize);
}
private bool AssertBackwardBoundary(ulong current, Entry* target, int level)
private bool AssertForwardBoundary(ulong currentOffset, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == (current + target->Size) >> size;
}
private bool AssertForwardBoundary(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == (current - target->Size) >> size;
int nextLevelSize = levelSizeArray[level] * 4;
ulong targetTail = target->Position - (ulong)m_Buffer + target->Size - 1;
return (currentOffset / (ulong)nextLevelSize) == (targetTail / (ulong)nextLevelSize);
}
/// <summary>
@@ -94,12 +88,12 @@ namespace StgSharp.HighPerformance.Memory
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(size, originLevel);
RemoveFromBucket(originLevel, segment, (BucketNode*)entry->Position);
while (originLevel >= entry->Level)
do
{
originLevel = entry->Level;
while (true)
@@ -107,10 +101,15 @@ namespace StgSharp.HighPerformance.Memory
// backward merging - simplified to single attempt
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
// 计算当前块相对于分配空间基址的偏移量
ulong currentOffset = entry->Position - (ulong)m_Buffer;
if (prev == null ||
prev == _spareMemory ||
!AssertBackwardBoundary(entry->Position, prev, originLevel) ||
IsAdjacentTo(entry, prev) ||
!AssertBackwardBoundary(currentOffset, prev, originLevel) ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
@@ -135,10 +134,13 @@ namespace StgSharp.HighPerformance.Memory
// forward merging - simplified to single attempt
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
ulong currentOffset = entry->Position - (ulong)m_Buffer;
if (next == null ||
next == _spareMemory ||
!AssertForwardBoundary(entry->Position + entry->Size, next, entry->Level) ||
IsAdjacentTo(entry, next) ||
!AssertForwardBoundary(currentOffset, next, entry->Level) ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(next->Size, originLevel), (BucketNode*)next->Position))
{
@@ -157,7 +159,7 @@ namespace StgSharp.HighPerformance.Memory
RecycleEntry(next);
}
entry->Level = GetLevelFromSize(entry->Size);
}
} while (originLevel < entry->Level);
}
private void RecycleEntry(Entry* entry)
@@ -171,45 +173,25 @@ namespace StgSharp.HighPerformance.Memory
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntrySpinLock(Entry* entry)
{
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
if (prev != null) {
prev->NextNear = entry->NextNear;
}
finally
{
ReleaseEntrySpinLock(entry);
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
return TryAcquireEntryBothLocks(entry);
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_entries.Free((nuint)entry);
}
/// <summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
-4
View File
@@ -66,8 +66,6 @@ namespace StgSharp.HighPerformance.Memory
_spareMemory->PreviousNear = EmptyHandle;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
_spareMemory->Size = (uint)byteSize;
}
@@ -89,8 +87,6 @@ namespace StgSharp.HighPerformance.Memory
_spareMemory->PreviousNear = EmptyHandle;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
_spareMemory->Size = (uint)byteSize;
}
StgSharp.Common/HighPerformance/Memory/ISlabAllocatable.cs
-40
View File
@@ -1,40 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="ISlabAllocatable.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public interface ISlabAllocatable { }
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs
+12 -36
View File
@@ -41,7 +41,6 @@ namespace StgSharp.HighPerformance.Memory
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private readonly BufferExpansionLock _lock = new();
private BufferStack<nuint> _buffers;
private BufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
@@ -74,42 +73,20 @@ namespace StgSharp.HighPerformance.Memory
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
while (true)
nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _currentCapacity)
{
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
nuint curIndex = Volatile.Read(ref _currentIndex);
_lock.EnterMetaDataRead();
try
{
if (curIndex >= _currentCapacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
nuint cap = Volatile.Read(ref _currentCapacity);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
_ = Interlocked.Exchange(ref _currentIndex, 0);
}
finally
{
_lock.ExitExpansionProcess();
}
}
curIndex = Volatile.Read(ref _currentIndex);
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
public override void Dispose()
@@ -118,7 +95,6 @@ namespace StgSharp.HighPerformance.Memory
while (_buffers.TryPop(out nuint buffer)) {
NativeMemory.Free((void*)buffer);
}
_lock.Dispose();
_buffers.Dispose();
_recycle.Dispose();
}
StgSharp.Common/HighPerformance/Memory/UnmanagedMemoryHandle.cs
-100
View File
@@ -1,100 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryHandle.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public unsafe struct UnmanagedMemoryHandle
{
public readonly byte* BufferHandle, AllocatorHandle;
public readonly nuint Size;
public UnmanagedMemoryHandle(byte* pointer, byte* handle, nuint s)
{
BufferHandle = pointer;
AllocatorHandle = handle;
Size = s;
}
public readonly ref byte this[nuint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref Unsafe.AddByteOffset(ref Unsafe.AsRef<byte>(BufferHandle), index);
}
public readonly Enumerator GetEnumerator()
{
return new Enumerator(this);
}
public ref struct Enumerator : IEnumerator<byte>
{
private nuint _index = nuint.MaxValue;
private UnmanagedMemoryHandle _handle;
internal Enumerator(UnmanagedMemoryHandle span)
{
_handle = span;
}
public ref byte RefCurrent => ref _handle[_index];
public readonly void Dispose() { }
public bool MoveNext()
{
_index++;
return _index < _handle.Size;
}
public void Reset()
{
_index = 0;
}
object IEnumerator.Current => RefCurrent;
byte IEnumerator<byte>.Current => RefCurrent;
}
}
}
StgSharp.Common/HighPerformance/Memory/UnmanagedMemoryPool.BucketTLSF.cs
-52
View File
@@ -1,52 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.BucketTLSF.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class UnmanagedMemoryPool
{
private struct Entry : ISlabAllocatable
{
public Entry* NextNear, PreviousNear;
}
}
}
StgSharp.Common/HighPerformance/Memory/UnmanagedMemoryPool.cs
-51
View File
@@ -1,51 +0,0 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="UnmanagedMemoryPool.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Concurrent;
using System.Drawing;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Transactions;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class UnmanagedMemoryPool
{
private byte* m_Buffer;
private SlabAllocator<Entry> _nodes = new(64);
public
}
}