604 lines
23 KiB
Plaintext
604 lines
23 KiB
Plaintext
//-----------------------------------------------------------------------
|
|
// -----------------------------------------------------------------------
|
|
// file="HLSFAllocator.BucketNode.cs"
|
|
// Project: StgSharp
|
|
// AuthorGroup: Nitload Space
|
|
// Copyright (c) Nitload Space. All rights reserved.
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person
|
|
// obtaining a copy of this software and associated documentation
|
|
// files (the “Software”), to deal in the Software without restriction,
|
|
// including without limitation the rights to use, copy, modify, merge,
|
|
// publish, distribute, sublicense, and/or sell copies of the Software,
|
|
// and to permit persons to whom the Software is furnished to do so,
|
|
// subject to the following conditions:
|
|
//
|
|
// The above copyright notice and
|
|
// this permission notice shall be included in all copies
|
|
// or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED “AS IS”,
|
|
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
|
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
|
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
|
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
|
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
//
|
|
// -----------------------------------------------------------------------
|
|
// -----------------------------------------------------------------------
|
|
using System;
|
|
using System.Runtime.CompilerServices;
|
|
using System.Runtime.InteropServices;
|
|
using System.Threading;
|
|
|
|
namespace StgSharp.HighPerformance.Memory
|
|
{
|
|
public unsafe partial class HybridLayerSegregatedFitAllocator
|
|
{
|
|
|
|
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
|
{
|
|
// 兼容性包装：获取两个锁
|
|
if (node != null)
|
|
{
|
|
BucketNodeOperations.AcquirePrevLock(node);
|
|
BucketNodeOperations.AcquireNextLock(node);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Initialize bucket system (called in constructor)
|
|
/// </summary>
|
|
private void InitializeBucketSystem()
|
|
{
|
|
_bucketHeads = new ulong[levelSize.Length * 3];
|
|
}
|
|
|
|
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
|
{
|
|
if (node == null || node->EntryRef == null) {
|
|
return;
|
|
}
|
|
|
|
int index = (3 * levelIndex) + sizeIndex;
|
|
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
|
{
|
|
node->IsInBucket = true;
|
|
node->Magic = BucketNode.MAGIC_NUMBER;
|
|
return;
|
|
} else
|
|
{
|
|
while (true)
|
|
{
|
|
// 对新节点只需要获取NextLock，因为我们要设置它的NextLevel
|
|
node->AcquireNextLock();
|
|
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
|
node->NextLevel = head;
|
|
BucketNode* headNode = (BucketNode*)head;
|
|
|
|
try
|
|
{
|
|
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
|
{
|
|
// 对头节点只需要获取PrevLock，因为我们要设置它的PreviousLevel
|
|
if (headNode != null && !headNode->TryAcquirePrevLock())
|
|
{
|
|
// reset state and retry
|
|
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
try
|
|
{
|
|
node->IsInBucket = true;
|
|
node->Magic = BucketNode.MAGIC_NUMBER;
|
|
if (headNode != null) {
|
|
headNode->PreviousLevel = (ulong)node;
|
|
}
|
|
return;
|
|
}
|
|
finally
|
|
{
|
|
if (headNode != null) {
|
|
headNode->ReleasePrevLock();
|
|
}
|
|
}
|
|
} else
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
node->ReleaseNextLock();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
|
{
|
|
// 兼容性包装：释放两个锁
|
|
if (node != null)
|
|
{
|
|
node->ReleaseBothLocks();
|
|
}
|
|
}
|
|
|
|
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
|
{
|
|
int index = (3 * levelIndex) + sizeIndex;
|
|
while (true)
|
|
{
|
|
// 检查是否是栈顶节点
|
|
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
|
{
|
|
// 栈顶节点直接使用TryPopLevel
|
|
if (TryPopLevel(levelIndex, sizeIndex, out _))
|
|
{
|
|
return;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
|
|
// 中间或底部节点的删除操作
|
|
// 获取当前节点的前后指针（只读操作，暂时不加锁）
|
|
ulong p = Volatile.Read(ref node->PreviousLevel);
|
|
ulong n = Volatile.Read(ref node->NextLevel);
|
|
BucketNode* previous = (BucketNode*)p;
|
|
BucketNode* next = (BucketNode*)n;
|
|
|
|
|
|
// 按照地址顺序获取锁，避免死锁
|
|
BucketNode* firstLock = null, secondLock = null, thirdLock = null;
|
|
bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false;
|
|
|
|
if (previous != null && next != null)
|
|
{
|
|
// 中间节点：需要修改前驱的next和后继的prev
|
|
if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next)
|
|
{
|
|
firstLock = previous;
|
|
needPrevNextLock = true;
|
|
secondLock = node;
|
|
needNodeBothLocks = true;
|
|
thirdLock = next;
|
|
needNextPrevLock = true;
|
|
} else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node)
|
|
{
|
|
firstLock = previous;
|
|
needPrevNextLock = true;
|
|
secondLock = next;
|
|
needNextPrevLock = true;
|
|
thirdLock = node;
|
|
needNodeBothLocks = true;
|
|
} else // other cases
|
|
{
|
|
firstLock = next;
|
|
needNextPrevLock = true;
|
|
secondLock = previous;
|
|
needPrevNextLock = true;
|
|
thirdLock = node;
|
|
needNodeBothLocks = true;
|
|
}
|
|
} else if (previous != null)
|
|
{
|
|
// 尾节点：只需要修改前驱的next
|
|
if ((ulong)previous < (ulong)node)
|
|
{
|
|
firstLock = previous;
|
|
needPrevNextLock = true;
|
|
secondLock = node;
|
|
needNodeBothLocks = true;
|
|
} else
|
|
{
|
|
firstLock = node;
|
|
needNodeBothLocks = true;
|
|
secondLock = previous;
|
|
needPrevNextLock = true;
|
|
}
|
|
} else
|
|
{
|
|
// 应该不会到这里，因为头节点已经在上面处理了
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
|
|
|
|
// 按顺序获取锁
|
|
bool acquired = false;
|
|
try
|
|
{
|
|
if (firstLock != null)
|
|
{
|
|
if (firstLock == previous && needPrevNextLock)
|
|
{
|
|
if (!firstLock->TryAcquireNextLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (firstLock == next && needNextPrevLock)
|
|
{
|
|
if (!firstLock->TryAcquirePrevLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (firstLock == node && needNodeBothLocks)
|
|
{
|
|
if (!firstLock->TryAcquireBothLocks())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (secondLock != null)
|
|
{
|
|
if (secondLock == previous && needPrevNextLock)
|
|
{
|
|
if (!secondLock->TryAcquireNextLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (secondLock == next && needNextPrevLock)
|
|
{
|
|
if (!secondLock->TryAcquirePrevLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (secondLock == node && needNodeBothLocks)
|
|
{
|
|
if (!secondLock->TryAcquireBothLocks())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (thirdLock != null)
|
|
{
|
|
if (thirdLock == previous && needPrevNextLock)
|
|
{
|
|
if (!thirdLock->TryAcquireNextLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (thirdLock == next && needNextPrevLock)
|
|
{
|
|
if (!thirdLock->TryAcquirePrevLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
} else if (thirdLock == node && needNodeBothLocks)
|
|
{
|
|
if (!thirdLock->TryAcquireBothLocks())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
|
|
acquired = true;
|
|
|
|
|
|
// 执行链表操作
|
|
if (previous != null)
|
|
{
|
|
previous->NextLevel = n;
|
|
}
|
|
if (next != null) {
|
|
next->PreviousLevel = p;
|
|
}
|
|
node->NextLevel = EmptyHandle;
|
|
node->PreviousLevel = EmptyHandle;
|
|
node->IsInBucket = false;
|
|
return;
|
|
}
|
|
finally
|
|
{
|
|
if (acquired)
|
|
{
|
|
// 释放锁
|
|
if (thirdLock == previous && needPrevNextLock)
|
|
{
|
|
thirdLock->ReleaseNextLock();
|
|
} else if (thirdLock == next && needNextPrevLock)
|
|
{
|
|
thirdLock->ReleasePrevLock();
|
|
} else if (thirdLock == node && needNodeBothLocks) {
|
|
thirdLock->ReleaseBothLocks();
|
|
}
|
|
|
|
if (secondLock == previous && needPrevNextLock)
|
|
{
|
|
secondLock->ReleaseNextLock();
|
|
} else if (secondLock == next && needNextPrevLock)
|
|
{
|
|
secondLock->ReleasePrevLock();
|
|
} else if (secondLock == node && needNodeBothLocks) {
|
|
secondLock->ReleaseBothLocks();
|
|
}
|
|
|
|
if (firstLock == previous && needPrevNextLock)
|
|
{
|
|
firstLock->ReleaseNextLock();
|
|
} else if (firstLock == next && needNextPrevLock)
|
|
{
|
|
firstLock->ReleasePrevLock();
|
|
} else if (firstLock == node && needNodeBothLocks) {
|
|
firstLock->ReleaseBothLocks();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
|
{
|
|
// 兼容性包装：尝试获取两个锁
|
|
return node == null || node->TryAcquireBothLocks();
|
|
}
|
|
|
|
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
|
{
|
|
int index = (3 * levelIndex) + sizeIndex;
|
|
while (true)
|
|
{
|
|
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
|
BucketNode* head = (BucketNode*)h;
|
|
|
|
if (head == null)
|
|
{
|
|
node = null;
|
|
return false;
|
|
}
|
|
|
|
|
|
// 对头节点获取NextLock，因为我们要读取它的NextLevel
|
|
if (!head->TryAcquireNextLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
|
|
ulong n = Volatile.Read(ref head->NextLevel);
|
|
BucketNode* next = (BucketNode*)n;
|
|
|
|
try
|
|
{
|
|
// 对next节点获取PrevLock，因为我们要修改它的PreviousLevel
|
|
if (next != null && !next->TryAcquirePrevLock())
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
|
{
|
|
node = head;
|
|
node->IsInBucket = false;
|
|
node->NextLevel = EmptyHandle;
|
|
if (next != null) {
|
|
next->PreviousLevel = EmptyHandle;
|
|
}
|
|
return true;
|
|
} else
|
|
{
|
|
Thread.Sleep(0);
|
|
continue;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (next != null) {
|
|
next->ReleasePrevLock();
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
head->ReleaseNextLock();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
|
/// allocation area (user-overwritable region)
|
|
/// </summary>
|
|
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
|
internal struct BucketNode
|
|
{
|
|
|
|
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
|
|
|
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
|
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
|
[FieldOffset(36)] internal int NextLock; // 后继指针锁（控制NextLevel指针操作）
|
|
|
|
// 双锁设计：分离前后指针操作
|
|
[FieldOffset(32)] internal int PrevLock; // 前驱指针锁（控制PreviousLevel指针操作）
|
|
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
|
|
|
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
|
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// BucketNode操作的静态方法集合
|
|
/// </summary>
|
|
private static class BucketNodeOperations
|
|
{
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsNullOrEmpty(BucketNode* node)
|
|
{
|
|
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
|
}
|
|
|
|
#region Lock Operations
|
|
|
|
// 前驱锁操作（控制PreviousLevel字段）
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool TryAcquirePrevLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return true;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void AcquirePrevLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (Volatile.Read(ref node->PrevLock) != thread)
|
|
{
|
|
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
|
Thread.Sleep(0);
|
|
}
|
|
}
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void ReleasePrevLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (Volatile.Read(ref node->PrevLock) == thread) {
|
|
Volatile.Write(ref node->PrevLock, 0);
|
|
}
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsPrevLockHeld(BucketNode* node) => node != null &&
|
|
Volatile.Read(ref node->PrevLock) != 0;
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
|
Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId;
|
|
|
|
// 后继锁操作（控制NextLevel字段）
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool TryAcquireNextLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return true;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void AcquireNextLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (Volatile.Read(ref node->NextLock) != thread)
|
|
{
|
|
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
|
Thread.Sleep(0);
|
|
}
|
|
}
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void ReleaseNextLock(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
int thread = Environment.CurrentManagedThreadId;
|
|
if (Volatile.Read(ref node->NextLock) == thread) {
|
|
Volatile.Write(ref node->NextLock, 0);
|
|
}
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsNextLockHeld(BucketNode* node) => node != null &&
|
|
Volatile.Read(ref node->NextLock) != 0;
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null &&
|
|
Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId;
|
|
|
|
// 双锁组合操作
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static bool TryAcquireBothLocks(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return true;
|
|
}
|
|
|
|
if (TryAcquirePrevLock(node))
|
|
{
|
|
if (TryAcquireNextLock(node))
|
|
{
|
|
return true;
|
|
} else
|
|
{
|
|
ReleasePrevLock(node); // 回滚
|
|
return false;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void ReleaseBothLocks(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
ReleasePrevLock(node);
|
|
ReleaseNextLock(node);
|
|
}
|
|
|
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
|
public static void AcquireBothLocks(BucketNode* node)
|
|
{
|
|
if (node == null) {
|
|
return;
|
|
}
|
|
|
|
AcquirePrevLock(node);
|
|
AcquireNextLock(node);
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
|
|
}
|
|
}
|