add from giga-device, new branch added

This commit is contained in:
Nitload Youmu
2025-08-21 16:51:25 +08:00
commit 43bbc06bbb
1296 changed files with 264186 additions and 0 deletions
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.AcquireBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.ReleaseBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
return BucketNodeOperations.TryAcquireBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,160 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->SetState(EntryState.Allocated);
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->SetState(EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->SetState(EntryState.ThreadOccupied);
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->Level = GetLevelFromSize((uint)blockSize);
// 初始化双锁为未锁定状态
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,286 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.AcquireBothLocks(entry);
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.ReleaseBothLocks(entry);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
return EntryOperations.TryAcquireBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,459 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.AcquireBothLocks(entry);
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.ReleaseBothLocks(entry);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
return EntryOperations.TryAcquireBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
#region Entry Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
{
return entry == null;
}
// Entry 状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return entry != null &&
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
if (entry != null) {
Volatile.Write(ref entry->State, newState);
}
}
// Entry 状态便利方法
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
// Entry 前驱锁操作(控制PreviousNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
// Entry 后继锁操作(控制NextNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
if (TryAcquireEntryPrevLock(entry))
{
if (TryAcquireEntryNextLock(entry))
{
return true;
} else
{
ReleaseEntryPrevLock(entry); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
AcquireEntryPrevLock(entry);
AcquireEntryNextLock(entry);
}
#endregion
}
}
@@ -0,0 +1,160 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
SetEntryState(handle, EntryState.Allocated);
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
SetEntryState(handle, EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->SetState(EntryState.ThreadOccupied);
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->Level = GetLevelFromSize((uint)blockSize);
// 初始化双锁为未锁定状态
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,596 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.AcquireBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.ReleaseBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
return BucketNodeOperations.TryAcquireBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
/// <summary>
/// BucketNode操作的静态方法集合
/// </summary>
private static class BucketNodeOperations
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
#region Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquirePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquirePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleasePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsPrevLockHeld(BucketNode* node) => node != null &&
Volatile.Read(ref node->PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null &&
Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquireNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquireNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleaseNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNextLockHeld(BucketNode* node) => node != null &&
Volatile.Read(ref node->NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null &&
Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquireBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquirePrevLock(node))
{
if (TryAcquireNextLock(node))
{
return true;
} else
{
ReleasePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleaseBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleasePrevLock(node);
ReleaseNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquireBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquirePrevLock(node);
AcquireNextLock(node);
}
#endregion
}
}
}
@@ -0,0 +1,453 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
AcquireEntryBothLocks(entry);
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
return TryAcquireEntryBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
#region Entry Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
{
return entry == null;
}
// Entry 状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return entry != null &&
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
if (entry != null) {
Volatile.Write(ref entry->State, newState);
}
}
// Entry 状态便利方法
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
// Entry 前驱锁操作(控制PreviousNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
// Entry 后继锁操作(控制NextNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
if (TryAcquireEntryPrevLock(entry))
{
if (TryAcquireEntryNextLock(entry))
{
return true;
} else
{
ReleaseEntryPrevLock(entry); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
AcquireEntryPrevLock(entry);
AcquireEntryNextLock(entry);
}
#endregion
}
}
@@ -0,0 +1,90 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (GetEntryState(e) != EntryState.Allocated) {
return;
}
SetEntryState(e, EntryState.ThreadOccupied);
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
SetEntryState(mergedEntry, EntryState.Empty);
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,123 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->SetState(EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->SetState(EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,286 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.AcquireBothLocks(entry);
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.ReleaseBothLocks(entry);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
return EntryOperations.TryAcquireBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.AcquireBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.ReleaseBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
return BucketNodeOperations.TryAcquireBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,570 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(firstLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(secondLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(thirdLock))
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(thirdLock);
} else if (thirdLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(thirdLock);
} else if (thirdLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(thirdLock);
}
if (secondLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(secondLock);
} else if (secondLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(secondLock);
} else if (secondLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(secondLock);
}
if (firstLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(firstLock);
} else if (firstLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(firstLock);
} else if (firstLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(firstLock);
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,123 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
SetEntryState(_spareMemory, EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
SetEntryState(_spareMemory, EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,570 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(firstLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(secondLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(thirdLock))
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(thirdLock);
} else if (thirdLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(thirdLock);
} else if (thirdLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(thirdLock);
}
if (secondLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(secondLock);
} else if (secondLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(secondLock);
} else if (secondLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(secondLock);
}
if (firstLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(firstLock);
} else if (firstLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(firstLock);
} else if (firstLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(firstLock);
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,569 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(firstLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(secondLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(thirdLock))
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(thirdLock);
} else if (thirdLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(thirdLock);
} else if (thirdLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(thirdLock);
}
if (secondLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(secondLock);
} else if (secondLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(secondLock);
} else if (secondLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(secondLock);
}
if (firstLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(firstLock);
} else if (firstLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(firstLock);
} else if (firstLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(firstLock);
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,90 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->GetState() != EntryState.Allocated) {
return;
}
e->SetState(EntryState.ThreadOccupied);
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->SetState(EntryState.Empty);
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,459 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.AcquireBothLocks(entry);
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
if (entry != null)
{
EntryOperations.ReleaseBothLocks(entry);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的静态方法
return EntryOperations.TryAcquireBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
#region Entry Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
{
return entry == null;
}
// Entry 状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return entry != null &&
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
if (entry != null) {
Volatile.Write(ref entry->State, newState);
}
}
// Entry 状态便利方法
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
// Entry 前驱锁操作(控制PreviousNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
// Entry 后继锁操作(控制NextNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
if (TryAcquireEntryPrevLock(entry))
{
if (TryAcquireEntryNextLock(entry))
{
return true;
} else
{
ReleaseEntryPrevLock(entry); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
AcquireEntryPrevLock(entry);
AcquireEntryNextLock(entry);
}
#endregion
}
}
@@ -0,0 +1,160 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
SetEntryState(handle, EntryState.Allocated);
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
SetEntryState(handle, EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
SetEntryState(handle, EntryState.ThreadOccupied);
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->Level = GetLevelFromSize((uint)blockSize);
// 初始化双锁为未锁定状态
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,453 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
AcquireEntryBothLocks(entry);
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
EntryOperations.GetState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
EntryOperations.GetState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
return TryAcquireEntryBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
#region Entry Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
{
return entry == null;
}
// Entry 状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return entry != null &&
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
if (entry != null) {
Volatile.Write(ref entry->State, newState);
}
}
// Entry 状态便利方法
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
// Entry 前驱锁操作(控制PreviousNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
// Entry 后继锁操作(控制NextNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
if (TryAcquireEntryPrevLock(entry))
{
if (TryAcquireEntryNextLock(entry))
{
return true;
} else
{
ReleaseEntryPrevLock(entry); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
AcquireEntryPrevLock(entry);
AcquireEntryNextLock(entry);
}
#endregion
}
}
@@ -0,0 +1,90 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->GetState() != EntryState.Allocated) {
return;
}
e->SetState(EntryState.ThreadOccupied);
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->SetState(EntryState.Empty);
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,160 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
SetEntryState(handle, EntryState.Allocated);
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
SetEntryState(handle, EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->SetState(EntryState.ThreadOccupied);
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->Level = GetLevelFromSize((uint)blockSize);
// 初始化双锁为未锁定状态
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,160 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->SetState(EntryState.Allocated);
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->SetState(EntryState.Allocated);
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->SetState(EntryState.ThreadOccupied);
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->Level = GetLevelFromSize((uint)blockSize);
// 初始化双锁为未锁定状态
handle->PrevLock = 0;
handle->NextLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,572 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,123 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->SetState(EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->SetState(EntryState.ThreadOccupied);
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->PrevLock = 0;
_spareMemory->NextLock = 0;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,596 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.AcquireBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
BucketNodeOperations.ReleaseBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用新的静态方法
return BucketNodeOperations.TryAcquireBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
/// <summary>
/// BucketNode操作的静态方法集合
/// </summary>
private static class BucketNodeOperations
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
#region Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquirePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquirePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleasePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsPrevLockHeld(BucketNode* node) => node != null &&
Volatile.Read(ref node->PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null &&
Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquireNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquireNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleaseNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNextLockHeld(BucketNode* node) => node != null &&
Volatile.Read(ref node->NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null &&
Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool TryAcquireBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquirePrevLock(node))
{
if (TryAcquireNextLock(node))
{
return true;
} else
{
ReleasePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void ReleaseBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleasePrevLock(node);
ReleaseNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void AcquireBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquirePrevLock(node);
AcquireNextLock(node);
}
#endregion
}
}
}
@@ -0,0 +1,453 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
AcquireEntryBothLocks(entry);
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return null;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = GetLevelFromSize(size);
int segment = DetermineSegmentIndex(size, originLevel);
if (!TryAcquireEntrySpinLock(entry))
{
return entry; // 无法获取锁,返回原Entry
}
try
{
// 设置状态为合并中
TrySetEntryState(entry, EntryState.Empty, EntryState.Merging);
// current Level merge
while (originLevel <= entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
// backward merging - 向前合并
while (true)
{
ulong p = entry->PreviousNear;
Entry* prev = (Entry*)p;
if (prev == null ||
GetEntryState(prev) != EntryState.Empty ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!AssertAlignBoundry(offset, prev, entry->Level) ||
!TryAcquireEntrySpinLock(prev))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
MergeEntry(entry, prev);
RecycleEntry(prev);
}
finally
{
ReleaseEntrySpinLock(prev);
}
}
// forward merging - 向后合并
while (true)
{
ulong n = entry->NextNear;
Entry* next = (Entry*)n;
if (next == null ||
GetEntryState(next) != EntryState.Empty ||
!IsAdjacentTo(entry, next) ||
entry->Level != next->Level ||
!AssertAlignBoundry(offset, next, entry->Level) ||
!TryAcquireEntrySpinLock(next))
{
break;
}
try
{
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
MergeEntry(entry, next);
RecycleEntry(next);
}
finally
{
ReleaseEntrySpinLock(next);
}
}
// 更新level信息
originLevel++;
if (originLevel > entry->Level)
{
break;
}
}
// 恢复状态为空
TrySetEntryState(entry, EntryState.Merging, EntryState.Empty);
return entry;
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
ReleaseEntryBothLocks(entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用HLSFAllocator的private static方法
return TryAcquireEntryBothLocks(entry);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 44)]
internal struct Entry
{
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
[FieldOffset(24)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
}
// 更新后的EntryState常量类 - 使用int类型
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int Merging = 3; // Currently being merged
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
#region Entry Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
{
return entry == null;
}
// Entry 状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
{
return entry != null &&
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryState(Entry* entry, int newState)
{
if (entry != null) {
Volatile.Write(ref entry->State, newState);
}
}
// Entry 状态便利方法
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
// Entry 前驱锁操作(控制PreviousNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryPrevLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->PrevLock) == thread) {
Volatile.Write(ref entry->PrevLock, 0);
}
}
// Entry 后继锁操作(控制NextNear字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryNextLock(Entry* entry)
{
if (entry == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref entry->NextLock) == thread) {
Volatile.Write(ref entry->NextLock, 0);
}
}
// Entry 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return true;
}
if (TryAcquireEntryPrevLock(entry))
{
if (TryAcquireEntryNextLock(entry))
{
return true;
} else
{
ReleaseEntryPrevLock(entry); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
ReleaseEntryPrevLock(entry);
ReleaseEntryNextLock(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireEntryBothLocks(Entry* entry)
{
if (entry == null) {
return;
}
AcquireEntryPrevLock(entry);
AcquireEntryNextLock(entry);
}
#endregion
}
}
@@ -0,0 +1,157 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->State = EntryState.Allocated;
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->State = EntryState.Allocated;
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->State = EntryState.ThreadOccupied;
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->SpinLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,356 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,356 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,91 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->State != EntryState.Allocated) {
return;
}
e->State = EntryState.ThreadOccupied;
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->State = EntryState.Empty;
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,356 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,119 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,550 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)- 改为int类型
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,157 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->State = EntryState.Allocated;
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->State = EntryState.Allocated;
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->State = EntryState.ThreadOccupied;
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->SpinLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,366 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,552 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(34)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)- 改为int类型
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,259 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,383 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,550 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)- 改为int类型
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,551 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,378 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用简单的volatile操作(调用者自觉模式)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
Volatile.Write(ref StateMask, newState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
byte current = Volatile.Read(ref PrevOpMask);
if (current != OpFree) {
return false;
}
Volatile.Write(ref PrevOpMask, OpOccupied);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
byte current = Volatile.Read(ref NextOpMask);
if (current != OpFree) {
return false;
}
Volatile.Write(ref NextOpMask, OpOccupied);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
byte current = Volatile.Read(ref SelfOpMask);
if (current != OpFree) {
return false;
}
Volatile.Write(ref SelfOpMask, OpOccupied);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,366 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,552 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(34)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)- 改为int类型
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,363 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,359 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,360 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
}
}
@@ -0,0 +1,363 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->GetState() != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,548 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)- 改为int类型
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
}
}
@@ -0,0 +1,383 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,259 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,364 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,91 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->State != EntryState.Allocated) {
return;
}
e->State = EntryState.ThreadOccupied;
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->State = EntryState.Empty;
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,360 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
}
}
@@ -0,0 +1,359 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,119 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,551 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:获取两个锁
if (node != null)
{
node->AcquirePrevLock();
node->AcquireNextLock();
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
node->AcquireNextLock();
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !headNode->TryAcquirePrevLock())
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
headNode->ReleasePrevLock();
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
node->ReleaseNextLock();
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:释放两个锁
if (node != null)
{
node->ReleaseBothLocks();
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
// 应该不会到这里,因为头节点已经在上面处理了
Thread.Sleep(0);
continue;
}
// 按顺序获取锁
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!firstLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!firstLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!firstLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!secondLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!secondLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!secondLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!thirdLock->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!thirdLock->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!thirdLock->TryAcquireBothLocks())
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
thirdLock->ReleaseNextLock();
} else if (thirdLock == next && needNextPrevLock)
{
thirdLock->ReleasePrevLock();
} else if (thirdLock == node && needNodeBothLocks) {
thirdLock->ReleaseBothLocks();
}
if (secondLock == previous && needPrevNextLock)
{
secondLock->ReleaseNextLock();
} else if (secondLock == next && needNextPrevLock)
{
secondLock->ReleasePrevLock();
} else if (secondLock == node && needNodeBothLocks) {
secondLock->ReleaseBothLocks();
}
if (firstLock == previous && needPrevNextLock)
{
firstLock->ReleaseNextLock();
} else if (firstLock == next && needNextPrevLock)
{
firstLock->ReleasePrevLock();
} else if (firstLock == node && needNodeBothLocks) {
firstLock->ReleaseBothLocks();
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 兼容性包装:尝试获取两个锁
return node == null || node->TryAcquireBothLocks();
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!head->TryAcquireNextLock())
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !next->TryAcquirePrevLock())
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
next->ReleasePrevLock();
}
}
}
finally
{
head->ReleaseNextLock();
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
// [FieldOffset(36)] 保留空间用于扩展
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Dual Lock Operations
// 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquirePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevLock()
{
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref PrevLock) == thread) {
Volatile.Write(ref PrevLock, 0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
// 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AcquireNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) != thread)
{
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextLock()
{
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
if (Volatile.Read(ref NextLock) == thread) {
Volatile.Write(ref NextLock, (short)0);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
// 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireBothLocks()
{
if (TryAcquirePrevLock())
{
if (TryAcquireNextLock())
{
return true;
} else
{
ReleasePrevLock(); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseBothLocks()
{
ReleasePrevLock();
ReleaseNextLock();
}
#endregion
}
private bool.
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,364 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
entry->ReleaseSelfOp();
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
return entry == null || entry->TryAcquireSelfOp();
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// 字节级原子操作 - 使用lock-free的自旋方式
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
byte current;
do
{
current = Volatile.Read(ref StateMask);
if (current != expectedState) {
return false;
}
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,157 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->State = EntryState.Allocated;
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->State = EntryState.Allocated;
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->State = EntryState.ThreadOccupied;
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->SpinLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,357 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,349 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
// 使用新的自操作锁代替SpinLock
if (entry != null)
{
while (!entry->TryAcquireSelfOp()) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,357 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations (replacing SpinLock)
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
[FieldOffset(28)] internal int _reserved; // Reserved for future use
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
#region Four Independent Byte Masks for Entry
// State constants
public const byte StateEmpty = 0;
public const byte StateAllocated = 1;
public const byte StateThreadOccupied = 2;
public const byte StateMerging = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Previous operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquirePrevOp()
{
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleasePrevOp()
{
Volatile.Write(ref PrevOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
// Next operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireNextOp()
{
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseNextOp()
{
Volatile.Write(ref NextOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsAllocated() => GetState() == StateAllocated;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMerging() => GetState() == StateMerging;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
#endregion
}
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
private static class EntryState
{
public const byte Allocated = 1; // Already allocated to user
public const byte Empty = 0; // Free, can be allocated
public const byte Merging = 3; // Currently being merged
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,347 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
// Four independent byte masks for atomic operations
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four Independent Byte Masks
// State constants
public const byte StateEmpty = 0;
public const byte StateInBucket = 1;
public const byte StateMarkedForRemoval = 2;
public const byte StateReserved = 3;
// Operation constants
public const byte OpFree = 0;
public const byte OpOccupied = 1;
// Atomic state operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => Volatile.Read(ref StateMask);
// Left operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
Volatile.Write(ref LeftOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
// Right operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
Volatile.Write(ref RightOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
// Self operation atomic methods
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
Volatile.Write(ref SelfOpMask, OpFree);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
// Convenience properties
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
#endregion
}
}
}
@@ -0,0 +1,91 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->State != EntryState.Allocated) {
return;
}
e->State = EntryState.ThreadOccupied;
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->State = EntryState.Empty;
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,119 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,157 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private BufferExpansionLock _spareLock = new();
public hlsfHandle Alloc(uint size)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
if (TryPopLevelForAllocation(i, s, out Entry* handle))
{
handle->State = EntryState.Allocated;
return new hlsfHandle(handle, size);
}
fallback:
i = level;
s = segmentIndex;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i >= levelSize.Length)
{
if (TryAllocateNew16MBBlock(out handle))
{
break;
}
Thread.Sleep(0);
goto fallback;
}
}
if (handle != null)
{
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
handle->State = EntryState.Allocated;
}
return new hlsfHandle(handle, size);
}
private void SliceMemory(Entry* e, int offset) { }
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew16MBBlock(out Entry* handle)
{
nuint pos = Volatile.Read(ref _spareMemory->Position);
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
{
handle = null;
return false; // Out of memory
}
// Try to allocate new Entry from slab allocator
try
{
handle = (Entry*)_nodes.Allocate();
handle->Position = pos;
handle->Size = (uint)blockSize;
handle->State = EntryState.ThreadOccupied;
handle->NextNear = EmptyHandle;
handle->PreviousNear = EmptyHandle;
handle->SpinLock = 0;
// Update spare memory position
_spareMemory->Position = newPos;
return true;
}
catch
{
handle = null;
return false;
}
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,404 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Four-byte mask for atomic operations
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region Four-Byte Mask Design
// Byte 0: Usage State (bits 0-7)
public const uint StateMask = 0x000000FF;
public const uint StateEmpty = 0x00000000;
public const uint StateInBucket = 0x00000001;
public const uint StateMarkedForRemoval = 0x00000002;
public const uint StateReserved = 0x00000003;
// Byte 1: Left Pointer Operation (bits 8-15)
public const uint LeftOpMask = 0x0000FF00;
public const uint LeftOpFree = 0x00000000;
public const uint LeftOpOccupied = 0x00000100;
// Byte 2: Right Pointer Operation (bits 16-23)
public const uint RightOpMask = 0x00FF0000;
public const uint RightOpFree = 0x00000000;
public const uint RightOpOccupied = 0x00010000;
// Byte 3: Self Operation (bits 24-31)
public const uint SelfOpMask = 0xFF000000;
public const uint SelfOpFree = 0x00000000;
public const uint SelfOpOccupied = 0x01000000;
// 快速访问器
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetState() => (byte)(Volatile.Read(ref Mask) & StateMask);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetLeftOpStatus() => (byte)((Volatile.Read(ref Mask) & LeftOpMask) >> 8);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetRightOpStatus() => (byte)((Volatile.Read(ref Mask) & RightOpMask) >> 16);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public byte GetSelfOpStatus() => (byte)((Volatile.Read(ref Mask) & SelfOpMask) >> 24);
// 原子状态操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TrySetState(byte expectedState, byte newState)
{
uint current = Volatile.Read(ref Mask);
uint currentState = current & StateMask;
if (currentState != expectedState) {
return false;
}
uint newMask = (current & ~StateMask) | newState;
return Interlocked.CompareExchange(ref Mask, newMask, current) == current;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireLeftOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
if ((current & LeftOpMask) != LeftOpFree) {
return false;
}
newMask = current | LeftOpOccupied;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseLeftOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
newMask = current & ~LeftOpMask;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireRightOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
if ((current & RightOpMask) != RightOpFree) {
return false;
}
newMask = current | RightOpOccupied;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseRightOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
newMask = current & ~RightOpMask;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryAcquireSelfOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
if ((current & SelfOpMask) != SelfOpFree) {
return false;
}
newMask = current | SelfOpOccupied;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ReleaseSelfOp()
{
uint current, newMask;
do
{
current = Volatile.Read(ref Mask);
newMask = current & ~SelfOpMask;
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
}
// 便利属性
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsInBucket() => GetState() == StateInBucket;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsEmpty() => GetState() == StateEmpty;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool HasAnyOperation() => (Volatile.Read(ref Mask) & (LeftOpMask | RightOpMask | SelfOpMask)) != 0;
#endregion
}
}
}
@@ -0,0 +1,261 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Linq.Expressions;
using System.Reflection.PortableExecutable;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Serialization;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
{
int size = (level * 2) + 6;
ulong align = current >> size;
return align == target->Position >> size && align == target->Size;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int DetermineSegmentIndex(uint size, int level)
{
return (int)size / levelSize[level];
}
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsAdjacentTo(Entry* current, Entry* other)
{
// Check if this immediately follows other
return other->Position + other->Size == current->Position ||
current->Position + current->Size == other->Position;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// stretagey, any fail in acquiring spin lock will end a merging process.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
Entry* result = entry;
uint size = entry->Size;
int originLevel = -1;
int segment = DetermineSegmentIndex(size, originLevel);
if (TryAcquireEntrySpinLock(entry)) {
return;
}
// current Level merge
while (originLevel != entry->Level)
{
ulong offset = entry->Position - (ulong)m_Buffer;
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
}
// forward merging
while (true)
{
ulong n = entry->NextNear;
Entry* near = (Entry*)n;
if (near == null ||
near->State != EntryState.Empty ||
!IsAdjacentTo(entry, near) ||
entry->Level != near->Level ||
!AssertAlignBoundry(offset, near, entry->Level) ||
!TryAcquireEntrySpinLock(near))
{
break;
}
}
}
}
private void MergeEntry(Entry* remain, Entry* other)
{
if (other == null) {
return;
}
// Determine merged starting position and size
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
uint newSize = remain->Size + other->Size;
// Update current Entry
remain->Position = newPosition;
remain->Size = newSize;
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry-> PreviousNear = EmptyHandle;
entry-> Position = EmptyHandle;
_nodes.Free((nuint)entry);
}
private static void ReleaseEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
}
}
/// <summary>
/// Remove node from position linked list
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
AcquireEntrySpinLock(entry);
try
{
Entry* prev = (Entry*)entry->PreviousNear;
Entry* next = (Entry*)entry->NextNear;
if (prev != null) {
prev->NextNear = entry->NextNear;
}
if (next != null) {
next->PreviousNear = entry->PreviousNear;
}
// Clear node pointers
entry->NextNear = EmptyHandle;
entry->PreviousNear = EmptyHandle;
_nodes.Free((nuint)entry);
}
finally
{
ReleaseEntrySpinLock(entry);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireEntrySpinLock(Entry* entry)
{
int thread = Environment.CurrentManagedThreadId;
return entry == null ||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
0) == 0) ||
Volatile.Read(ref entry->SpinLock) == thread;
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
[FieldOffset(28)] internal int State; // Node state
[FieldOffset(32)] internal uint Size; // Block size information
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
[FieldOffset(0)] public nuint Position; // Memory block position
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(ulong index)
{
return index == EmptyHandle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmptyIndex(Entry* entry)
{
return entry == null;
}
/// <summary>
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
/// state)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void MergeWith(Entry* other) { }
}
private static class EntryState
{
public const int Allocated = 1; // Already allocated to user
public const int Empty = 0; // Free, can be allocated
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
}
}
}
@@ -0,0 +1,91 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
AcquireEntrySpinLock(e);
try
{
if (e->State != EntryState.Allocated) {
return;
}
e->State = EntryState.ThreadOccupied;
// Step 4: Try merge with adjacent free blocks and remove them from buckets
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
// Step 5: Calculate final level and segment for the merged block
uint finalSize = mergedEntry->Size;
int finalLevel = GetLevelFromSize(finalSize);
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Step 6: Set the merged entry state to Empty (ready to be reused)
mergedEntry->State = EntryState.Empty;
// Step 7: Push the merged entry back to appropriate bucket
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
// Note: If mergedEntry is different from original e, the original e
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
}
finally
{
// Step 8: Always release the spin lock
ReleaseEntrySpinLock(e);
}
}
}
}
@@ -0,0 +1,119 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
private static readonly nuint EmptyHandle = (nuint)(void*)null;
// Bucket system field - directly in main allocator class
private ulong[] _bucketHeads;
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _nodes;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
// 初始化桶系统
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 8 bytes.");
}
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
InitializeBucketSystem();
// 初始化spareMemory
_spareMemory = (Entry*)_nodes.Allocate();
_spareMemory->State = EntryState.ThreadOccupied;
_spareMemory->NextNear = EmptyHandle;
_spareMemory->Position = (nuint)m_Buffer;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_nodes.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,276 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
{
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
AcquireBucketNodeSpinLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
if (!TryAcquireBucketNodeSpinLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->Mask = true;
node->Magic = BucketNode.MAGIC_NUMBER;
headNode->PreviousLevel = (ulong)node;
return;
}
finally
{
ReleaseBucketNodeSpinLock(headNode);
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// top of stack
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
{
_ = TryPopLevel(levelIndex, sizeIndex, out _);
} else
{
AcquireBucketNodeSpinLock(node);
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
try
{
if (next is null)
{
// bottom of stack
if (!TryAcquireBucketNodeSpinLock(previous))
{
Thread.Sleep(0);
continue;
}
previous->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
{
// middle of stack
Thread.Sleep(0);
continue;
}
previous ->NextLevel = n;
next->PreviousLevel = p;
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
return;
}
finally
{
ReleaseBucketNodeSpinLock(node);
ReleaseBucketNodeSpinLock(previous);
ReleaseBucketNodeSpinLock(next);
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
int thread = Environment.CurrentManagedThreadId;
return (node == null) ||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
0) == 0) ||
(Volatile.Read(ref node->SpinLock) == thread);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
BucketNode* next = null;
try
{
if (head == null)
{
node = null;
return false;
}
if (!TryAcquireBucketNodeSpinLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
next = (BucketNode*)n;
if (!TryAcquireBucketNodeSpinLock(next))
{
Thread.Sleep(0);
continue;
}
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->Mask = false;
node->NextLevel = EmptyHandle;
next->PreviousLevel = EmptyHandle;
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeSpinLock(head);
ReleaseBucketNodeSpinLock(next);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 36)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(28)] internal uint Mask; // Whether in bucket
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != MAGIC_NUMBER;
}
#region node mask
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
private bool BucketNodeIsInBucket
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (Mask & InBucket) != 0;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set => Mask |= value ? InBucket : 0;
}
#endregion
}
}
}
@@ -0,0 +1,523 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
/// <summary>
/// 根据锁类型释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
{
switch (lockType)
{
case 0:
ReleaseBucketNodePrevLock(node);
break; // PrevLock
case 1:
ReleaseBucketNodeBothLocks(node);
break; // BothLocks
case 2:
ReleaseBucketNodeNextLock(node);
break; // NextLock
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 直接释放,顺序不重要
if (previous != null)
{
ReleaseBucketNodeNextLock(previous);
}
if (node != null) {
ReleaseBucketNodeBothLocks(node);
}
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(无锁读取)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 直接抢锁策略:按地址顺序获取锁,避免死锁
if (!TryAcquireOrderedBucketLocks(previous, node, next))
{
Thread.Sleep(0);
continue;
}
try
{
// 验证链表结构仍然有效(防止ABA问题)
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
{
Thread.Sleep(0);
continue;
}
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序获取BucketNode锁,避免死锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 创建排序数组(最多3个非null节点)
Span<BucketNode*> nodes = stackalloc BucketNode*[3];
Span<byte> lockTypes = stackalloc byte[3]; // 0=prev, 1=both, 2=next
int count = 0;
if (previous != null)
{
nodes[count] = previous;
lockTypes[count] = 2; // previous需要NextLock
count++;
}
if (node != null)
{
nodes[count] = node;
lockTypes[count] = 1; // node需要BothLocks
count++;
}
if (next != null)
{
nodes[count] = next;
lockTypes[count] = 0; // next需要PrevLock
count++;
}
// 按地址排序(简单冒泡排序,最多3个元素)
for (int i = 0; i < count - 1; i++)
{
for (int j = 0; j < count - i - 1; j++)
{
if ((ulong)nodes[j] > (ulong)nodes[j + 1])
{
(nodes[j], nodes[j + 1]) = (nodes[j + 1], nodes[j]);
(lockTypes[j], lockTypes[j + 1]) = (lockTypes[j + 1], lockTypes[j]);
}
}
}
// 按顺序尝试获取锁
int acquired = 0;
for (int i = 0; i < count; i++)
{
bool success = lockTypes[i] switch
{
0 => TryAcquireBucketNodePrevLock(nodes[i]), // PrevLock
1 => TryAcquireBucketNodeBothLocks(nodes[i]), // BothLocks
2 => TryAcquireBucketNodeNextLock(nodes[i]), // NextLock
_ => false
};
if (!success)
{
// 回滚已获取的锁
for (int j = acquired - 1; j >= 0; j--)
{
ReleaseBucketNodeLockByType(nodes[j], lockTypes[j]);
}
return false;
}
acquired++;
}
return true;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,567 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(只读操作,暂时不加锁)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 按照地址顺序获取锁,避免死锁
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
if (previous != null && next != null)
{
// 中间节点:需要修改前驱的next和后继的prev
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
thirdLock = next;
needNextPrevLock = true;
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = next;
needNextPrevLock = true;
thirdLock = node;
needNodeBothLocks = true;
} else // other cases
{
firstLock = next;
needNextPrevLock = true;
secondLock = previous;
needPrevNextLock = true;
thirdLock = node;
needNodeBothLocks = true;
}
} else if (previous != null)
{
// 尾节点:只需要修改前驱的next
if ((ulong)previous < (ulong)node)
{
firstLock = previous;
needPrevNextLock = true;
secondLock = node;
needNodeBothLocks = true;
} else
{
firstLock = node;
needNodeBothLocks = true;
secondLock = previous;
needPrevNextLock = true;
}
} else
{
Thread.Sleep(0);
continue;
}
bool acquired = false;
try
{
if (firstLock != null)
{
if (firstLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(firstLock))
{
Thread.Sleep(0);
continue;
}
} else if (firstLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(firstLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (secondLock != null)
{
if (secondLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(secondLock))
{
Thread.Sleep(0);
continue;
}
} else if (secondLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(secondLock))
{
Thread.Sleep(0);
continue;
}
}
}
if (thirdLock != null)
{
if (thirdLock == previous && needPrevNextLock)
{
if (!TryAcquireBucketNodeNextLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == next && needNextPrevLock)
{
if (!TryAcquireBucketNodePrevLock(thirdLock))
{
Thread.Sleep(0);
continue;
}
} else if (thirdLock == node && needNodeBothLocks)
{
if (!TryAcquireBucketNodeBothLocks(thirdLock))
{
Thread.Sleep(0);
continue;
}
}
}
acquired = true;
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
if (acquired)
{
// 释放锁
if (thirdLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(thirdLock);
} else if (thirdLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(thirdLock);
} else if (thirdLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(thirdLock);
}
if (secondLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(secondLock);
} else if (secondLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(secondLock);
} else if (secondLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(secondLock);
}
if (firstLock == previous && needPrevNextLock)
{
ReleaseBucketNodeNextLock(firstLock);
} else if (firstLock == next && needNextPrevLock)
{
ReleaseBucketNodePrevLock(firstLock);
} else if (firstLock == node && needNodeBothLocks) {
ReleaseBucketNodeBothLocks(firstLock);
}
}
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,534 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
/// <summary>
/// 根据锁类型释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
{
switch (lockType)
{
case 0:
ReleaseBucketNodePrevLock(node);
break; // PrevLock
case 1:
ReleaseBucketNodeBothLocks(node);
break; // BothLocks
case 2:
ReleaseBucketNodeNextLock(node);
break; // NextLock
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 直接释放,顺序不重要
if (previous != null)
{
ReleaseBucketNodeNextLock(previous);
}
if (node != null) {
ReleaseBucketNodeBothLocks(node);
}
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(无锁读取)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 简化的地址排序锁获取策略
if (!TryAcquireOrderedBucketLocks(previous, node, next))
{
Thread.Sleep(0);
continue;
}
try
{
// 验证链表结构仍然有效(防止ABA问题)
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
{
Thread.Sleep(0);
continue;
}
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序获取BucketNode锁,避免死锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 创建排序数组(使用nuint数组存储地址和锁类型)
nuint prevAddr = (nuint)previous;
nuint nodeAddr = (nuint)node;
nuint nextAddr = (nuint)next;
// 使用简单的比较来确定锁的顺序
if (previous != null && node != null && next != null)
{
// 三个节点都存在,按地址排序
if (prevAddr < nodeAddr && nodeAddr < nextAddr)
{
// previous -> node -> next
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next);
} else if (prevAddr < nextAddr && nextAddr < nodeAddr)
{
// previous -> next -> node
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeBothLocks(node);
} else if (nodeAddr < prevAddr && prevAddr < nextAddr)
{
// node -> previous -> next
return TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodePrevLock(next);
} else if (nodeAddr < nextAddr && nextAddr < prevAddr)
{
// node -> next -> previous
return TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeNextLock(previous);
} else if (nextAddr < prevAddr && prevAddr < nodeAddr)
{
// next -> previous -> node
return TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node);
} else // nextAddr < nodeAddr && nodeAddr < prevAddr
{
// next -> node -> previous
return TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodeNextLock(previous);
}
} else if (previous != null && node != null)
{
// 只有previous和node
if (prevAddr < nodeAddr)
{
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
} else
{
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous);
}
} else if (node != null && next != null)
{
// 只有node和next
if (nodeAddr < nextAddr)
{
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next);
} else
{
return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node);
}
} else if (node != null)
{
// 只有node
return TryAcquireBucketNodeBothLocks(node);
}
return true; // 没有节点需要锁定
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}
@@ -0,0 +1,534 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private static void AcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
AcquireBucketNodeBothLocks(node);
}
/// <summary>
/// Initialize bucket system (called in constructor)
/// </summary>
private void InitializeBucketSystem()
{
_bucketHeads = new ulong[levelSize.Length * 3];
}
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
int index = (3 * levelIndex) + sizeIndex;
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
return;
} else
{
while (true)
{
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
AcquireBucketNodeNextLock(node);
ulong head = Volatile.Read(ref _bucketHeads[index]);
node->NextLevel = head;
BucketNode* headNode = (BucketNode*)head;
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
{
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
{
// reset state and retry
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
Thread.Sleep(0);
continue;
}
try
{
node->IsInBucket = true;
node->Magic = BucketNode.MAGIC_NUMBER;
if (headNode != null) {
headNode->PreviousLevel = (ulong)node;
}
return;
}
finally
{
if (headNode != null) {
ReleaseBucketNodePrevLock(headNode);
}
}
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
ReleaseBucketNodeNextLock(node);
}
}
}
}
/// <summary>
/// 根据锁类型释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
{
switch (lockType)
{
case 0:
ReleaseBucketNodePrevLock(node);
break; // PrevLock
case 1:
ReleaseBucketNodeBothLocks(node);
break; // BothLocks
case 2:
ReleaseBucketNodeNextLock(node);
break; // NextLock
}
}
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
ReleaseBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序释放BucketNode锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 直接释放,顺序不重要
if (previous != null)
{
ReleaseBucketNodeNextLock(previous);
}
if (node != null) {
ReleaseBucketNodeBothLocks(node);
}
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
// 检查是否是栈顶节点
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
{
// 栈顶节点直接使用TryPopLevel
if (TryPopLevel(levelIndex, sizeIndex, out _))
{
return;
}
continue;
}
// 中间或底部节点的删除操作
// 获取当前节点的前后指针(无锁读取)
ulong p = Volatile.Read(ref node->PreviousLevel);
ulong n = Volatile.Read(ref node->NextLevel);
BucketNode* previous = (BucketNode*)p;
BucketNode* next = (BucketNode*)n;
// 直接抢锁策略:按地址顺序获取锁,避免死锁
if (!TryAcquireOrderedBucketLocks(previous, node, next))
{
Thread.Sleep(0);
continue;
}
try
{
// 验证链表结构仍然有效(防止ABA问题)
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
{
Thread.Sleep(0);
continue;
}
// 执行链表操作
if (previous != null)
{
previous->NextLevel = n;
}
if (next != null) {
next->PreviousLevel = p;
}
node->NextLevel = EmptyHandle;
node->PreviousLevel = EmptyHandle;
node->IsInBucket = false;
return;
}
finally
{
ReleaseOrderedBucketLocks(previous, node, next);
}
}
}
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
{
// 使用HLSFAllocator的private static方法
return TryAcquireBucketNodeBothLocks(node);
}
/// <summary>
/// 按地址顺序获取BucketNode锁,避免死锁
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
{
// 创建排序数组(使用nuint数组存储地址和锁类型)
nuint prevAddr = (nuint)previous;
nuint nodeAddr = (nuint)node;
nuint nextAddr = (nuint)next;
// 使用简单的比较来确定锁的顺序
if (previous != null && node != null && next != null)
{
// 三个节点都存在,按地址排序
if (prevAddr < nodeAddr && nodeAddr < nextAddr)
{
// previous -> node -> next
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next);
} else if (prevAddr < nextAddr && nextAddr < nodeAddr)
{
// previous -> next -> node
return TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeBothLocks(node);
} else if (nodeAddr < prevAddr && prevAddr < nextAddr)
{
// node -> previous -> next
return TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodePrevLock(next);
} else if (nodeAddr < nextAddr && nextAddr < prevAddr)
{
// node -> next -> previous
return TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeNextLock(previous);
} else if (nextAddr < prevAddr && prevAddr < nodeAddr)
{
// next -> previous -> node
return TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeNextLock(previous) &&
TryAcquireBucketNodeBothLocks(node);
} else // nextAddr < nodeAddr && nodeAddr < prevAddr
{
// next -> node -> previous
return TryAcquireBucketNodePrevLock(next) &&
TryAcquireBucketNodeBothLocks(node) &&
TryAcquireBucketNodeNextLock(previous);
}
} else if (previous != null && node != null)
{
// 只有previous和node
if (prevAddr < nodeAddr)
{
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
} else
{
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous);
}
} else if (node != null && next != null)
{
// 只有node和next
if (nodeAddr < nextAddr)
{
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next);
} else
{
return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node);
}
} else if (node != null)
{
// 只有node
return TryAcquireBucketNodeBothLocks(node);
}
return true; // 没有节点需要锁定
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
int index = (3 * levelIndex) + sizeIndex;
while (true)
{
ulong h = Volatile.Read(ref _bucketHeads[index]);
BucketNode* head = (BucketNode*)h;
if (head == null)
{
node = null;
return false;
}
// 对头节点获取NextLock,因为我们要读取它的NextLevel
if (!TryAcquireBucketNodeNextLock(head))
{
Thread.Sleep(0);
continue;
}
ulong n = Volatile.Read(ref head->NextLevel);
BucketNode* next = (BucketNode*)n;
try
{
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
if (next != null && !TryAcquireBucketNodePrevLock(next))
{
Thread.Sleep(0);
continue;
}
try
{
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
{
node = head;
node->IsInBucket = false;
node->NextLevel = EmptyHandle;
if (next != null) {
next->PreviousLevel = EmptyHandle;
}
return true;
} else
{
Thread.Sleep(0);
continue;
}
}
finally
{
if (next != null) {
ReleaseBucketNodePrevLock(next);
}
}
}
finally
{
ReleaseBucketNodeNextLock(head);
}
}
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct BucketNode
{
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
// 双锁设计:分离前后指针操作
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
[FieldOffset(24)] internal uint Magic; // Magic number verification
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
}
#region BucketNode Operations
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsNullOrEmpty(BucketNode* node)
{
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
}
// BucketNode 前驱锁操作(控制PreviousLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) != thread)
{
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodePrevLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->PrevLock) == thread) {
Volatile.Write(ref node->PrevLock, 0);
}
}
// BucketNode 后继锁操作(控制NextLevel字段)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return true;
}
int thread = Environment.CurrentManagedThreadId;
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) != thread)
{
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
Thread.Sleep(0);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeNextLock(BucketNode* node)
{
if (node == null) {
return;
}
int thread = Environment.CurrentManagedThreadId;
if (Volatile.Read(ref node->NextLock) == thread) {
Volatile.Write(ref node->NextLock, 0);
}
}
// BucketNode 双锁组合操作
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return true;
}
if (TryAcquireBucketNodePrevLock(node))
{
if (TryAcquireBucketNodeNextLock(node))
{
return true;
} else
{
ReleaseBucketNodePrevLock(node); // 回滚
return false;
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
ReleaseBucketNodePrevLock(node);
ReleaseBucketNodeNextLock(node);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AcquireBucketNodeBothLocks(BucketNode* node)
{
if (node == null) {
return;
}
AcquireBucketNodePrevLock(node);
AcquireBucketNodeNextLock(node);
}
#endregion
}
}

Some files were not shown because too many files have changed in this diff Show More