add from giga-device, new branch added
This commit is contained in:
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.AcquireBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.ReleaseBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return BucketNodeOperations.TryAcquireBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->SetState(EntryState.Allocated);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->SetState(EntryState.Allocated);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->SetState(EntryState.ThreadOccupied);
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
|
||||
// 初始化双锁为未锁定状态
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.AcquireBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.ReleaseBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return EntryOperations.TryAcquireBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+459
@@ -0,0 +1,459 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.AcquireBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.ReleaseBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return EntryOperations.TryAcquireBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
#region Entry Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
// Entry 状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
|
||||
{
|
||||
return entry != null &&
|
||||
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryState(Entry* entry, int newState)
|
||||
{
|
||||
if (entry != null) {
|
||||
Volatile.Write(ref entry->State, newState);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 状态便利方法
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
|
||||
|
||||
// Entry 前驱锁操作(控制PreviousNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) == thread) {
|
||||
Volatile.Write(ref entry->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 后继锁操作(控制NextNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) == thread) {
|
||||
Volatile.Write(ref entry->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireEntryPrevLock(entry))
|
||||
{
|
||||
if (TryAcquireEntryNextLock(entry))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseEntryPrevLock(entry); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseEntryPrevLock(entry);
|
||||
ReleaseEntryNextLock(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireEntryPrevLock(entry);
|
||||
AcquireEntryNextLock(entry);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->SetState(EntryState.ThreadOccupied);
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
|
||||
// 初始化双锁为未锁定状态
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+596
@@ -0,0 +1,596 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.AcquireBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.ReleaseBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return BucketNodeOperations.TryAcquireBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode操作的静态方法集合
|
||||
/// </summary>
|
||||
private static class BucketNodeOperations
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquirePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquirePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleasePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsPrevLockHeld(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquireNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquireNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleaseNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNextLockHeld(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquireBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquirePrevLock(node))
|
||||
{
|
||||
if (TryAcquireNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleaseBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleasePrevLock(node);
|
||||
ReleaseNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquireBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquirePrevLock(node);
|
||||
AcquireNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
#region Entry Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
// Entry 状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
|
||||
{
|
||||
return entry != null &&
|
||||
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryState(Entry* entry, int newState)
|
||||
{
|
||||
if (entry != null) {
|
||||
Volatile.Write(ref entry->State, newState);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 状态便利方法
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
|
||||
|
||||
// Entry 前驱锁操作(控制PreviousNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) == thread) {
|
||||
Volatile.Write(ref entry->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 后继锁操作(控制NextNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) == thread) {
|
||||
Volatile.Write(ref entry->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireEntryPrevLock(entry))
|
||||
{
|
||||
if (TryAcquireEntryNextLock(entry))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseEntryPrevLock(entry); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseEntryPrevLock(entry);
|
||||
ReleaseEntryNextLock(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireEntryPrevLock(entry);
|
||||
AcquireEntryNextLock(entry);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (GetEntryState(e) != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
SetEntryState(e, EntryState.ThreadOccupied);
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
SetEntryState(mergedEntry, EntryState.Empty);
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->SetState(EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->SetState(EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+286
@@ -0,0 +1,286 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.AcquireBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.ReleaseBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return EntryOperations.TryAcquireBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.AcquireBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.ReleaseBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return BucketNodeOperations.TryAcquireBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+570
@@ -0,0 +1,570 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(thirdLock);
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(thirdLock);
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(thirdLock);
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(secondLock);
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(secondLock);
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(secondLock);
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(firstLock);
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(firstLock);
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(firstLock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
SetEntryState(_spareMemory, EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
SetEntryState(_spareMemory, EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+570
@@ -0,0 +1,570 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(thirdLock);
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(thirdLock);
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(thirdLock);
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(secondLock);
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(secondLock);
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(secondLock);
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(firstLock);
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(firstLock);
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(firstLock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+569
@@ -0,0 +1,569 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(thirdLock);
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(thirdLock);
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(thirdLock);
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(secondLock);
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(secondLock);
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(secondLock);
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(firstLock);
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(firstLock);
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(firstLock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->GetState() != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->SetState(EntryState.ThreadOccupied);
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->SetState(EntryState.Empty);
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+459
@@ -0,0 +1,459 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.AcquireBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
if (entry != null)
|
||||
{
|
||||
EntryOperations.ReleaseBothLocks(entry);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return EntryOperations.TryAcquireBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
#region Entry Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
// Entry 状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
|
||||
{
|
||||
return entry != null &&
|
||||
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryState(Entry* entry, int newState)
|
||||
{
|
||||
if (entry != null) {
|
||||
Volatile.Write(ref entry->State, newState);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 状态便利方法
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
|
||||
|
||||
// Entry 前驱锁操作(控制PreviousNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) == thread) {
|
||||
Volatile.Write(ref entry->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 后继锁操作(控制NextNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) == thread) {
|
||||
Volatile.Write(ref entry->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireEntryPrevLock(entry))
|
||||
{
|
||||
if (TryAcquireEntryNextLock(entry))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseEntryPrevLock(entry); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseEntryPrevLock(entry);
|
||||
ReleaseEntryNextLock(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireEntryPrevLock(entry);
|
||||
AcquireEntryNextLock(entry);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
SetEntryState(handle, EntryState.ThreadOccupied);
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
|
||||
// 初始化双锁为未锁定状态
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
EntryOperations.TrySetState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
EntryOperations.GetState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
EntryOperations.GetState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
EntryOperations.TrySetState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
#region Entry Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
// Entry 状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
|
||||
{
|
||||
return entry != null &&
|
||||
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryState(Entry* entry, int newState)
|
||||
{
|
||||
if (entry != null) {
|
||||
Volatile.Write(ref entry->State, newState);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 状态便利方法
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
|
||||
|
||||
// Entry 前驱锁操作(控制PreviousNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) == thread) {
|
||||
Volatile.Write(ref entry->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 后继锁操作(控制NextNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) == thread) {
|
||||
Volatile.Write(ref entry->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireEntryPrevLock(entry))
|
||||
{
|
||||
if (TryAcquireEntryNextLock(entry))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseEntryPrevLock(entry); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseEntryPrevLock(entry);
|
||||
ReleaseEntryNextLock(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireEntryPrevLock(entry);
|
||||
AcquireEntryNextLock(entry);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->GetState() != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->SetState(EntryState.ThreadOccupied);
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = MergeAndRemoveFromBuckets(e);
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->SetState(EntryState.Empty);
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by MergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
SetEntryState(handle, EntryState.Allocated);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->SetState(EntryState.ThreadOccupied);
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
|
||||
// 初始化双锁为未锁定状态
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->SetState(EntryState.Allocated);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->SetState(EntryState.Allocated);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->SetState(EntryState.ThreadOccupied);
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
|
||||
// 初始化双锁为未锁定状态
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+572
@@ -0,0 +1,572 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->SetState(EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->SetState(EntryState.ThreadOccupied);
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+596
@@ -0,0 +1,596 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.AcquireBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
BucketNodeOperations.ReleaseBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用新的静态方法
|
||||
return BucketNodeOperations.TryAcquireBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode操作的静态方法集合
|
||||
/// </summary>
|
||||
private static class BucketNodeOperations
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquirePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquirePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleasePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsPrevLockHeld(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquireNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquireNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleaseNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNextLockHeld(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
||||
Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool TryAcquireBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquirePrevLock(node))
|
||||
{
|
||||
if (TryAcquireNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void ReleaseBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleasePrevLock(node);
|
||||
ReleaseNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void AcquireBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquirePrevLock(node);
|
||||
AcquireNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+453
@@ -0,0 +1,453 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private Entry* MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
|
||||
if (!TryAcquireEntrySpinLock(entry))
|
||||
{
|
||||
return entry; // 无法获取锁,返回原Entry
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 设置状态为合并中
|
||||
TrySetEntryState(entry, EntryState.Empty, EntryState.Merging);
|
||||
|
||||
// current Level merge
|
||||
while (originLevel <= entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
|
||||
// backward merging - 向前合并
|
||||
while (true)
|
||||
{
|
||||
ulong p = entry->PreviousNear;
|
||||
Entry* prev = (Entry*)p;
|
||||
if (prev == null ||
|
||||
GetEntryState(prev) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, prev) ||
|
||||
entry->Level != prev->Level ||
|
||||
!AssertAlignBoundry(offset, prev, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)prev->Position);
|
||||
MergeEntry(entry, prev);
|
||||
RecycleEntry(prev);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
}
|
||||
|
||||
// forward merging - 向后合并
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
if (next == null ||
|
||||
GetEntryState(next) != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, next) ||
|
||||
entry->Level != next->Level ||
|
||||
!AssertAlignBoundry(offset, next, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)next->Position);
|
||||
MergeEntry(entry, next);
|
||||
RecycleEntry(next);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新level信息
|
||||
originLevel++;
|
||||
if (originLevel > entry->Level)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 恢复状态为空
|
||||
TrySetEntryState(entry, EntryState.Merging, EntryState.Empty);
|
||||
return entry;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 44)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(40)] internal int NextLock; // 后继指针锁(控制NextNear指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作(和BucketNode保持一致)
|
||||
[FieldOffset(36)] internal int PrevLock; // 前驱指针锁(控制PreviousNear指针操作)
|
||||
[FieldOffset(32)] internal int State; // Node state (using int for atomic operations)
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 使用int类型
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int Merging = 3; // Currently being merged
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
#region Entry Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmptyEntryIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
// Entry 状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TrySetEntryState(Entry* entry, int expectedState, int newState)
|
||||
{
|
||||
return entry != null &&
|
||||
Interlocked.CompareExchange(ref entry->State, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetEntryState(Entry* entry) => entry != null ? Volatile.Read(ref entry->State) : 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryState(Entry* entry, int newState)
|
||||
{
|
||||
if (entry != null) {
|
||||
Volatile.Write(ref entry->State, newState);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 状态便利方法
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryEmpty(Entry* entry) => GetEntryState(entry) == EntryState.Empty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryAllocated(Entry* entry) => GetEntryState(entry) == EntryState.Allocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryThreadOccupied(Entry* entry) => GetEntryState(entry) == EntryState.ThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsEntryMerging(Entry* entry) => GetEntryState(entry) == EntryState.Merging;
|
||||
|
||||
// Entry 前驱锁操作(控制PreviousNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryPrevLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->PrevLock) == thread) {
|
||||
Volatile.Write(ref entry->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 后继锁操作(控制NextNear字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref entry->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryNextLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref entry->NextLock) == thread) {
|
||||
Volatile.Write(ref entry->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Entry 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireEntryPrevLock(entry))
|
||||
{
|
||||
if (TryAcquireEntryNextLock(entry))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseEntryPrevLock(entry); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseEntryPrevLock(entry);
|
||||
ReleaseEntryNextLock(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntryBothLocks(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireEntryPrevLock(entry);
|
||||
AcquireEntryNextLock(entry);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->State = EntryState.ThreadOccupied;
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
|
||||
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->State = EntryState.Empty;
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+550
@@ -0,0 +1,550 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)- 改为int类型
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+552
@@ -0,0 +1,552 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(34)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)- 改为int类型
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+550
@@ -0,0 +1,550 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)- 改为int类型
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+551
@@ -0,0 +1,551 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+378
@@ -0,0 +1,378 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用简单的volatile操作(调用者自觉模式)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Volatile.Write(ref StateMask, newState);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
byte current = Volatile.Read(ref PrevOpMask);
|
||||
if (current != OpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Volatile.Write(ref PrevOpMask, OpOccupied);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
byte current = Volatile.Read(ref NextOpMask);
|
||||
if (current != OpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Volatile.Write(ref NextOpMask, OpOccupied);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
byte current = Volatile.Read(ref SelfOpMask);
|
||||
if (current != OpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
Volatile.Write(ref SelfOpMask, OpOccupied);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+552
@@ -0,0 +1,552 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(34)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)- 改为int类型
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+363
@@ -0,0 +1,363 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+363
@@ -0,0 +1,363 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->GetState() != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+548
@@ -0,0 +1,548 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)] // 增加到40字节以容纳两个int锁
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)- 改为int类型
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+364
@@ -0,0 +1,364 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->State = EntryState.ThreadOccupied;
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
|
||||
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->State = EntryState.Empty;
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+551
@@ -0,0 +1,551 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:获取两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->AcquirePrevLock();
|
||||
node->AcquireNextLock();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
node->AcquireNextLock();
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !headNode->TryAcquirePrevLock())
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
headNode->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
node->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:释放两个锁
|
||||
if (node != null)
|
||||
{
|
||||
node->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
// 应该不会到这里,因为头节点已经在上面处理了
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// 按顺序获取锁
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!firstLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!firstLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!firstLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!secondLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!secondLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!secondLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!thirdLock->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!thirdLock->TryAcquireBothLocks())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
thirdLock->ReleaseNextLock();
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
thirdLock->ReleasePrevLock();
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
thirdLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
secondLock->ReleaseNextLock();
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
secondLock->ReleasePrevLock();
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
secondLock->ReleaseBothLocks();
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
firstLock->ReleaseNextLock();
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
firstLock->ReleasePrevLock();
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
firstLock->ReleaseBothLocks();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 兼容性包装:尝试获取两个锁
|
||||
return node == null || node->TryAcquireBothLocks();
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!head->TryAcquireNextLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !next->TryAcquirePrevLock())
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
next->ReleasePrevLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
head->ReleaseNextLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(34)] internal short NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
// [FieldOffset(36)] 保留空间用于扩展
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Dual Lock Operations
|
||||
|
||||
// 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquirePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevLock()
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref PrevLock) == thread) {
|
||||
Volatile.Write(ref PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeld() => Volatile.Read(ref PrevLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevLockHeldByCurrentThread() => Volatile.Read(ref PrevLock) == Environment.CurrentManagedThreadId;
|
||||
|
||||
// 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF); // 使用低15位
|
||||
return Interlocked.CompareExchange(ref NextLock, thread, (short)0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void AcquireNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref NextLock, thread, (short)0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextLock()
|
||||
{
|
||||
short thread = (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
if (Volatile.Read(ref NextLock) == thread) {
|
||||
Volatile.Write(ref NextLock, (short)0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeld() => Volatile.Read(ref NextLock) != 0;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextLockHeldByCurrentThread() => Volatile.Read(ref NextLock) == (short)(Environment.CurrentManagedThreadId & 0x7FFF);
|
||||
|
||||
// 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireBothLocks()
|
||||
{
|
||||
if (TryAcquirePrevLock())
|
||||
{
|
||||
if (TryAcquireNextLock())
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleasePrevLock(); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseBothLocks()
|
||||
{
|
||||
ReleasePrevLock();
|
||||
ReleaseNextLock();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
private bool.
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+364
@@ -0,0 +1,364 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, segment, (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
entry->ReleaseSelfOp();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
return entry == null || entry->TryAcquireSelfOp();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+357
@@ -0,0 +1,357 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
// 使用新的自操作锁代替SpinLock
|
||||
if (entry != null)
|
||||
{
|
||||
while (!entry->TryAcquireSelfOp()) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+357
@@ -0,0 +1,357 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(26)] internal byte NextOpMask; // Next pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(25)] internal byte PrevOpMask; // Previous pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(27)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations (replacing SpinLock)
|
||||
[FieldOffset(24)] internal byte StateMask; // Entry state (Empty=0, Allocated=1, ThreadOccupied=2, Merging=3)
|
||||
|
||||
[FieldOffset(28)] internal int _reserved; // Reserved for future use
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
#region Four Independent Byte Masks for Entry
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateAllocated = 1;
|
||||
public const byte StateThreadOccupied = 2;
|
||||
public const byte StateMerging = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Previous operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquirePrevOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref PrevOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleasePrevOp()
|
||||
{
|
||||
Volatile.Write(ref PrevOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsPrevOpOccupied() => Volatile.Read(ref PrevOpMask) == OpOccupied;
|
||||
|
||||
// Next operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireNextOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref NextOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseNextOp()
|
||||
{
|
||||
Volatile.Write(ref NextOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsNextOpOccupied() => Volatile.Read(ref NextOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsAllocated() => GetState() == StateAllocated;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsThreadOccupied() => GetState() == StateThreadOccupied;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMerging() => GetState() == StateMerging;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsPrevOpOccupied() || IsNextOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
// 更新后的EntryState常量类 - 与新的字节掩码设计兼容
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const byte Allocated = 1; // Already allocated to user
|
||||
public const byte Empty = 0; // Free, can be allocated
|
||||
public const byte Merging = 3; // Currently being merged
|
||||
public const byte ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref StateMask, newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->State = EntryState.ThreadOccupied;
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
|
||||
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->State = EntryState.Empty;
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+404
@@ -0,0 +1,404 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Four-byte mask for atomic operations
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four-Byte Mask Design
|
||||
|
||||
// Byte 0: Usage State (bits 0-7)
|
||||
public const uint StateMask = 0x000000FF;
|
||||
public const uint StateEmpty = 0x00000000;
|
||||
public const uint StateInBucket = 0x00000001;
|
||||
public const uint StateMarkedForRemoval = 0x00000002;
|
||||
public const uint StateReserved = 0x00000003;
|
||||
|
||||
// Byte 1: Left Pointer Operation (bits 8-15)
|
||||
public const uint LeftOpMask = 0x0000FF00;
|
||||
public const uint LeftOpFree = 0x00000000;
|
||||
public const uint LeftOpOccupied = 0x00000100;
|
||||
|
||||
// Byte 2: Right Pointer Operation (bits 16-23)
|
||||
public const uint RightOpMask = 0x00FF0000;
|
||||
public const uint RightOpFree = 0x00000000;
|
||||
public const uint RightOpOccupied = 0x00010000;
|
||||
|
||||
// Byte 3: Self Operation (bits 24-31)
|
||||
public const uint SelfOpMask = 0xFF000000;
|
||||
public const uint SelfOpFree = 0x00000000;
|
||||
public const uint SelfOpOccupied = 0x01000000;
|
||||
|
||||
// 快速访问器
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => (byte)(Volatile.Read(ref Mask) & StateMask);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetLeftOpStatus() => (byte)((Volatile.Read(ref Mask) & LeftOpMask) >> 8);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetRightOpStatus() => (byte)((Volatile.Read(ref Mask) & RightOpMask) >> 16);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetSelfOpStatus() => (byte)((Volatile.Read(ref Mask) & SelfOpMask) >> 24);
|
||||
|
||||
// 原子状态操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
uint current = Volatile.Read(ref Mask);
|
||||
uint currentState = current & StateMask;
|
||||
|
||||
if (currentState != expectedState) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint newMask = (current & ~StateMask) | newState;
|
||||
return Interlocked.CompareExchange(ref Mask, newMask, current) == current;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
if ((current & LeftOpMask) != LeftOpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
newMask = current | LeftOpOccupied;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
newMask = current & ~LeftOpMask;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
if ((current & RightOpMask) != RightOpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
newMask = current | RightOpOccupied;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
newMask = current & ~RightOpMask;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
if ((current & SelfOpMask) != SelfOpFree) {
|
||||
return false;
|
||||
}
|
||||
|
||||
newMask = current | SelfOpOccupied;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
uint current, newMask;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref Mask);
|
||||
newMask = current & ~SelfOpMask;
|
||||
} while (Interlocked.CompareExchange(ref Mask, newMask, current) != current);
|
||||
}
|
||||
|
||||
// 便利属性
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => (Volatile.Read(ref Mask) & (LeftOpMask | RightOpMask | SelfOpMask)) != 0;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->State = EntryState.ThreadOccupied;
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
|
||||
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->State = EntryState.Empty;
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->Mask = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->Mask = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(28)] internal uint Mask; // Whether in bucket
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region node mask
|
||||
|
||||
public const uint AllOperating = 0b_11100000_00000000_00000000_00000111;
|
||||
public const uint Empty = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint InBucket = 0b_10000000_00000000_00000000_00000000;
|
||||
public const uint LeftOperating = 0b_01000000_00000000_00000000_00000100;
|
||||
public const uint ReadyForRemove = 0b_01000000_00000000_00000000_00000000;
|
||||
public const uint RightOperating = 0b_00100000_00000000_00000000_00000001;
|
||||
public const uint SelfOperating = 0b_00000000_00000000_00000000_00000010;
|
||||
|
||||
private bool BucketNodeIsInBucket
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (Mask & InBucket) != 0;
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => Mask |= value ? InBucket : 0;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+523
@@ -0,0 +1,523 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 根据锁类型释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
|
||||
{
|
||||
switch (lockType)
|
||||
{
|
||||
case 0:
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
break; // PrevLock
|
||||
case 1:
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
break; // BothLocks
|
||||
case 2:
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
break; // NextLock
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 直接释放,顺序不重要
|
||||
if (previous != null)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(previous);
|
||||
}
|
||||
if (node != null) {
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(无锁读取)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
// 直接抢锁策略:按地址顺序获取锁,避免死锁
|
||||
if (!TryAcquireOrderedBucketLocks(previous, node, next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 验证链表结构仍然有效(防止ABA问题)
|
||||
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseOrderedBucketLocks(previous, node, next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序获取BucketNode锁,避免死锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 创建排序数组(最多3个非null节点)
|
||||
Span<BucketNode*> nodes = stackalloc BucketNode*[3];
|
||||
Span<byte> lockTypes = stackalloc byte[3]; // 0=prev, 1=both, 2=next
|
||||
int count = 0;
|
||||
|
||||
if (previous != null)
|
||||
{
|
||||
nodes[count] = previous;
|
||||
lockTypes[count] = 2; // previous需要NextLock
|
||||
count++;
|
||||
}
|
||||
if (node != null)
|
||||
{
|
||||
nodes[count] = node;
|
||||
lockTypes[count] = 1; // node需要BothLocks
|
||||
count++;
|
||||
}
|
||||
if (next != null)
|
||||
{
|
||||
nodes[count] = next;
|
||||
lockTypes[count] = 0; // next需要PrevLock
|
||||
count++;
|
||||
}
|
||||
|
||||
|
||||
// 按地址排序(简单冒泡排序,最多3个元素)
|
||||
for (int i = 0; i < count - 1; i++)
|
||||
{
|
||||
for (int j = 0; j < count - i - 1; j++)
|
||||
{
|
||||
if ((ulong)nodes[j] > (ulong)nodes[j + 1])
|
||||
{
|
||||
(nodes[j], nodes[j + 1]) = (nodes[j + 1], nodes[j]);
|
||||
(lockTypes[j], lockTypes[j + 1]) = (lockTypes[j + 1], lockTypes[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 按顺序尝试获取锁
|
||||
int acquired = 0;
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
bool success = lockTypes[i] switch
|
||||
{
|
||||
0 => TryAcquireBucketNodePrevLock(nodes[i]), // PrevLock
|
||||
1 => TryAcquireBucketNodeBothLocks(nodes[i]), // BothLocks
|
||||
2 => TryAcquireBucketNodeNextLock(nodes[i]), // NextLock
|
||||
_ => false
|
||||
};
|
||||
|
||||
if (!success)
|
||||
{
|
||||
// 回滚已获取的锁
|
||||
for (int j = acquired - 1; j >= 0; j--)
|
||||
{
|
||||
ReleaseBucketNodeLockByType(nodes[j], lockTypes[j]);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
acquired++;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+567
@@ -0,0 +1,567 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(只读操作,暂时不加锁)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
|
||||
// 按照地址顺序获取锁,避免死锁
|
||||
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
||||
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
||||
|
||||
if (previous != null && next != null)
|
||||
{
|
||||
// 中间节点:需要修改前驱的next和后继的prev
|
||||
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
thirdLock = next;
|
||||
needNextPrevLock = true;
|
||||
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = next;
|
||||
needNextPrevLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else // other cases
|
||||
{
|
||||
firstLock = next;
|
||||
needNextPrevLock = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
thirdLock = node;
|
||||
needNodeBothLocks = true;
|
||||
}
|
||||
} else if (previous != null)
|
||||
{
|
||||
// 尾节点:只需要修改前驱的next
|
||||
if ((ulong)previous < (ulong)node)
|
||||
{
|
||||
firstLock = previous;
|
||||
needPrevNextLock = true;
|
||||
secondLock = node;
|
||||
needNodeBothLocks = true;
|
||||
} else
|
||||
{
|
||||
firstLock = node;
|
||||
needNodeBothLocks = true;
|
||||
secondLock = previous;
|
||||
needPrevNextLock = true;
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool acquired = false;
|
||||
try
|
||||
{
|
||||
if (firstLock != null)
|
||||
{
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (firstLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(firstLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (secondLock != null)
|
||||
{
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (secondLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(secondLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thirdLock != null)
|
||||
{
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodeNextLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
} else if (thirdLock == node && needNodeBothLocks)
|
||||
{
|
||||
if (!TryAcquireBucketNodeBothLocks(thirdLock))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
acquired = true;
|
||||
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (acquired)
|
||||
{
|
||||
// 释放锁
|
||||
if (thirdLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(thirdLock);
|
||||
} else if (thirdLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(thirdLock);
|
||||
} else if (thirdLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(thirdLock);
|
||||
}
|
||||
|
||||
if (secondLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(secondLock);
|
||||
} else if (secondLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(secondLock);
|
||||
} else if (secondLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(secondLock);
|
||||
}
|
||||
|
||||
if (firstLock == previous && needPrevNextLock)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(firstLock);
|
||||
} else if (firstLock == next && needNextPrevLock)
|
||||
{
|
||||
ReleaseBucketNodePrevLock(firstLock);
|
||||
} else if (firstLock == node && needNodeBothLocks) {
|
||||
ReleaseBucketNodeBothLocks(firstLock);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+534
@@ -0,0 +1,534 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 根据锁类型释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
|
||||
{
|
||||
switch (lockType)
|
||||
{
|
||||
case 0:
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
break; // PrevLock
|
||||
case 1:
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
break; // BothLocks
|
||||
case 2:
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
break; // NextLock
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 直接释放,顺序不重要
|
||||
if (previous != null)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(previous);
|
||||
}
|
||||
if (node != null) {
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(无锁读取)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
// 简化的地址排序锁获取策略
|
||||
if (!TryAcquireOrderedBucketLocks(previous, node, next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 验证链表结构仍然有效(防止ABA问题)
|
||||
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseOrderedBucketLocks(previous, node, next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序获取BucketNode锁,避免死锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 创建排序数组(使用nuint数组存储地址和锁类型)
|
||||
nuint prevAddr = (nuint)previous;
|
||||
nuint nodeAddr = (nuint)node;
|
||||
nuint nextAddr = (nuint)next;
|
||||
|
||||
|
||||
// 使用简单的比较来确定锁的顺序
|
||||
if (previous != null && node != null && next != null)
|
||||
{
|
||||
// 三个节点都存在,按地址排序
|
||||
if (prevAddr < nodeAddr && nodeAddr < nextAddr)
|
||||
{
|
||||
// previous -> node -> next
|
||||
return TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodePrevLock(next);
|
||||
} else if (prevAddr < nextAddr && nextAddr < nodeAddr)
|
||||
{
|
||||
// previous -> next -> node
|
||||
return TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeBothLocks(node);
|
||||
} else if (nodeAddr < prevAddr && prevAddr < nextAddr)
|
||||
{
|
||||
// node -> previous -> next
|
||||
return TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodePrevLock(next);
|
||||
} else if (nodeAddr < nextAddr && nextAddr < prevAddr)
|
||||
{
|
||||
// node -> next -> previous
|
||||
return TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeNextLock(previous);
|
||||
} else if (nextAddr < prevAddr && prevAddr < nodeAddr)
|
||||
{
|
||||
// next -> previous -> node
|
||||
return TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodeBothLocks(node);
|
||||
} else // nextAddr < nodeAddr && nodeAddr < prevAddr
|
||||
{
|
||||
// next -> node -> previous
|
||||
return TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodeNextLock(previous);
|
||||
}
|
||||
} else if (previous != null && node != null)
|
||||
{
|
||||
// 只有previous和node
|
||||
if (prevAddr < nodeAddr)
|
||||
{
|
||||
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
|
||||
} else
|
||||
{
|
||||
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous);
|
||||
}
|
||||
} else if (node != null && next != null)
|
||||
{
|
||||
// 只有node和next
|
||||
if (nodeAddr < nextAddr)
|
||||
{
|
||||
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next);
|
||||
} else
|
||||
{
|
||||
return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
} else if (node != null)
|
||||
{
|
||||
// 只有node
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
return true; // 没有节点需要锁定
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
+534
@@ -0,0 +1,534 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// 对新节点只需要获取NextLock,因为我们要设置它的NextLevel
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
// 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 根据锁类型释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
|
||||
{
|
||||
switch (lockType)
|
||||
{
|
||||
case 0:
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
break; // PrevLock
|
||||
case 1:
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
break; // BothLocks
|
||||
case 2:
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
break; // NextLock
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 直接释放,顺序不重要
|
||||
if (previous != null)
|
||||
{
|
||||
ReleaseBucketNodeNextLock(previous);
|
||||
}
|
||||
if (node != null) {
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// 检查是否是栈顶节点
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
// 栈顶节点直接使用TryPopLevel
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 中间或底部节点的删除操作
|
||||
// 获取当前节点的前后指针(无锁读取)
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
// 直接抢锁策略:按地址顺序获取锁,避免死锁
|
||||
if (!TryAcquireOrderedBucketLocks(previous, node, next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 验证链表结构仍然有效(防止ABA问题)
|
||||
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 执行链表操作
|
||||
if (previous != null)
|
||||
{
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseOrderedBucketLocks(previous, node, next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
// 使用HLSFAllocator的private static方法
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 按地址顺序获取BucketNode锁,避免死锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
// 创建排序数组(使用nuint数组存储地址和锁类型)
|
||||
nuint prevAddr = (nuint)previous;
|
||||
nuint nodeAddr = (nuint)node;
|
||||
nuint nextAddr = (nuint)next;
|
||||
|
||||
|
||||
// 使用简单的比较来确定锁的顺序
|
||||
if (previous != null && node != null && next != null)
|
||||
{
|
||||
// 三个节点都存在,按地址排序
|
||||
if (prevAddr < nodeAddr && nodeAddr < nextAddr)
|
||||
{
|
||||
// previous -> node -> next
|
||||
return TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodePrevLock(next);
|
||||
} else if (prevAddr < nextAddr && nextAddr < nodeAddr)
|
||||
{
|
||||
// previous -> next -> node
|
||||
return TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeBothLocks(node);
|
||||
} else if (nodeAddr < prevAddr && prevAddr < nextAddr)
|
||||
{
|
||||
// node -> previous -> next
|
||||
return TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodePrevLock(next);
|
||||
} else if (nodeAddr < nextAddr && nextAddr < prevAddr)
|
||||
{
|
||||
// node -> next -> previous
|
||||
return TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeNextLock(previous);
|
||||
} else if (nextAddr < prevAddr && prevAddr < nodeAddr)
|
||||
{
|
||||
// next -> previous -> node
|
||||
return TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodeBothLocks(node);
|
||||
} else // nextAddr < nodeAddr && nodeAddr < prevAddr
|
||||
{
|
||||
// next -> node -> previous
|
||||
return TryAcquireBucketNodePrevLock(next) &&
|
||||
TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodeNextLock(previous);
|
||||
}
|
||||
} else if (previous != null && node != null)
|
||||
{
|
||||
// 只有previous和node
|
||||
if (prevAddr < nodeAddr)
|
||||
{
|
||||
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
|
||||
} else
|
||||
{
|
||||
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous);
|
||||
}
|
||||
} else if (node != null && next != null)
|
||||
{
|
||||
// 只有node和next
|
||||
if (nodeAddr < nextAddr)
|
||||
{
|
||||
return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next);
|
||||
} else
|
||||
{
|
||||
return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
} else if (node != null)
|
||||
{
|
||||
// 只有node
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
return true; // 没有节点需要锁定
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
// 对头节点获取NextLock,因为我们要读取它的NextLevel
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
try
|
||||
{
|
||||
// 对next节点获取PrevLock,因为我们要修改它的PreviousLevel
|
||||
if (next != null && !TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作)
|
||||
|
||||
// 双锁设计:分离前后指针操作
|
||||
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 双锁组合操作
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user