//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="HLSFAllocator.BucketNode.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the “Software”), to deal in the Software without restriction, // including without limitation the rights to use, copy, modify, merge, // publish, distribute, sublicense, and/or sell copies of the Software, // and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and // this permission notice shall be included in all copies // or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED “AS IS”, // WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, // INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. // IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, // ARISING FROM, OUT OF OR IN CONNECTION WITH // THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; namespace StgSharp.HighPerformance.Memory { public unsafe partial class HybridLayerSegregatedFitAllocator { private static void AcquireBucketNodeSpinLock(BucketNode* node) { // 使用新的静态方法 BucketNodeOperations.AcquireBothLocks(node); } /// /// Initialize bucket system (called in constructor) /// private void InitializeBucketSystem() { _bucketHeads = new ulong[levelSize.Length * 3]; } private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node) { if (node == null || node->EntryRef == null) { return; } int index = (3 * levelIndex) + sizeIndex; if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle) { node->IsInBucket = true; node->Magic = BucketNode.MAGIC_NUMBER; return; } else { while (true) { // 对新节点只需要获取NextLock,因为我们要设置它的NextLevel node->AcquireNextLock(); ulong head = Volatile.Read(ref _bucketHeads[index]); node->NextLevel = head; BucketNode* headNode = (BucketNode*)head; try { if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head) { // 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel if (headNode != null && !headNode->TryAcquirePrevLock()) { // reset state and retry _ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node); Thread.Sleep(0); continue; } try { node->IsInBucket = true; node->Magic = BucketNode.MAGIC_NUMBER; if (headNode != null) { headNode->PreviousLevel = (ulong)node; } return; } finally { if (headNode != null) { headNode->ReleasePrevLock(); } } } else { Thread.Sleep(0); continue; } } finally { node->ReleaseNextLock(); } } } } private static void ReleaseBucketNodeSpinLock(BucketNode* node) { // 使用新的静态方法 BucketNodeOperations.ReleaseBothLocks(node); } private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node) { int index = (3 * levelIndex) + sizeIndex; while (true) { // 检查是否是栈顶节点 if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node) { // 栈顶节点直接使用TryPopLevel if (TryPopLevel(levelIndex, sizeIndex, out _)) { return; } continue; } // 中间或底部节点的删除操作 // 获取当前节点的前后指针(只读操作,暂时不加锁) ulong p = Volatile.Read(ref node->PreviousLevel); ulong n = Volatile.Read(ref node->NextLevel); BucketNode* previous = (BucketNode*)p; BucketNode* next = (BucketNode*)n; // 按照地址顺序获取锁,避免死锁 BucketNode* firstLock = null, secondLock = null, thirdLock = null; bool needPrevNextLock = false, needNodeBothLocks = false, needNextPrevLock = false; if (previous != null && next != null) { // 中间节点:需要修改前驱的next和后继的prev if ((ulong)previous < (ulong)node && (ulong)node < (ulong)next) { firstLock = previous; needPrevNextLock = true; secondLock = node; needNodeBothLocks = true; thirdLock = next; needNextPrevLock = true; } else if ((ulong)previous < (ulong)next && (ulong)next < (ulong)node) { firstLock = previous; needPrevNextLock = true; secondLock = next; needNextPrevLock = true; thirdLock = node; needNodeBothLocks = true; } else // other cases { firstLock = next; needNextPrevLock = true; secondLock = previous; needPrevNextLock = true; thirdLock = node; needNodeBothLocks = true; } } else if (previous != null) { // 尾节点:只需要修改前驱的next if ((ulong)previous < (ulong)node) { firstLock = previous; needPrevNextLock = true; secondLock = node; needNodeBothLocks = true; } else { firstLock = node; needNodeBothLocks = true; secondLock = previous; needPrevNextLock = true; } } else { // 应该不会到这里,因为头节点已经在上面处理了 Thread.Sleep(0); continue; } // 按顺序获取锁 bool acquired = false; try { if (firstLock != null) { if (firstLock == previous && needPrevNextLock) { if (!firstLock->TryAcquireNextLock()) { Thread.Sleep(0); continue; } } else if (firstLock == next && needNextPrevLock) { if (!firstLock->TryAcquirePrevLock()) { Thread.Sleep(0); continue; } } else if (firstLock == node && needNodeBothLocks) { if (!firstLock->TryAcquireBothLocks()) { Thread.Sleep(0); continue; } } } if (secondLock != null) { if (secondLock == previous && needPrevNextLock) { if (!secondLock->TryAcquireNextLock()) { Thread.Sleep(0); continue; } } else if (secondLock == next && needNextPrevLock) { if (!secondLock->TryAcquirePrevLock()) { Thread.Sleep(0); continue; } } else if (secondLock == node && needNodeBothLocks) { if (!secondLock->TryAcquireBothLocks()) { Thread.Sleep(0); continue; } } } if (thirdLock != null) { if (thirdLock == previous && needPrevNextLock) { if (!thirdLock->TryAcquireNextLock()) { Thread.Sleep(0); continue; } } else if (thirdLock == next && needNextPrevLock) { if (!thirdLock->TryAcquirePrevLock()) { Thread.Sleep(0); continue; } } else if (thirdLock == node && needNodeBothLocks) { if (!thirdLock->TryAcquireBothLocks()) { Thread.Sleep(0); continue; } } } acquired = true; // 执行链表操作 if (previous != null) { previous->NextLevel = n; } if (next != null) { next->PreviousLevel = p; } node->NextLevel = EmptyHandle; node->PreviousLevel = EmptyHandle; node->IsInBucket = false; return; } finally { if (acquired) { // 释放锁 if (thirdLock == previous && needPrevNextLock) { thirdLock->ReleaseNextLock(); } else if (thirdLock == next && needNextPrevLock) { thirdLock->ReleasePrevLock(); } else if (thirdLock == node && needNodeBothLocks) { thirdLock->ReleaseBothLocks(); } if (secondLock == previous && needPrevNextLock) { secondLock->ReleaseNextLock(); } else if (secondLock == next && needNextPrevLock) { secondLock->ReleasePrevLock(); } else if (secondLock == node && needNodeBothLocks) { secondLock->ReleaseBothLocks(); } if (firstLock == previous && needPrevNextLock) { firstLock->ReleaseNextLock(); } else if (firstLock == next && needNextPrevLock) { firstLock->ReleasePrevLock(); } else if (firstLock == node && needNodeBothLocks) { firstLock->ReleaseBothLocks(); } } } } } private static bool TryAcquireBucketNodeSpinLock(BucketNode* node) { // 使用新的静态方法 return BucketNodeOperations.TryAcquireBothLocks(node); } private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node) { int index = (3 * levelIndex) + sizeIndex; while (true) { ulong h = Volatile.Read(ref _bucketHeads[index]); BucketNode* head = (BucketNode*)h; if (head == null) { node = null; return false; } // 对头节点获取NextLock,因为我们要读取它的NextLevel if (!head->TryAcquireNextLock()) { Thread.Sleep(0); continue; } ulong n = Volatile.Read(ref head->NextLevel); BucketNode* next = (BucketNode*)n; try { // 对next节点获取PrevLock,因为我们要修改它的PreviousLevel if (next != null && !next->TryAcquirePrevLock()) { Thread.Sleep(0); continue; } try { if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h) { node = head; node->IsInBucket = false; node->NextLevel = EmptyHandle; if (next != null) { next->PreviousLevel = EmptyHandle; } return true; } else { Thread.Sleep(0); continue; } } finally { if (next != null) { next->ReleasePrevLock(); } } } finally { head->ReleaseNextLock(); } } } /// /// BucketNode structure - specialized node for bucket linked lists Stored in main /// allocation area (user-overwritable region) /// [StructLayout(LayoutKind.Explicit, Size = 40)] internal struct BucketNode { public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification [FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry [FieldOffset(28)] internal bool IsInBucket; // Whether in bucket [FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作) // 双锁设计:分离前后指针操作 [FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作) [FieldOffset(24)] internal uint Magic; // Magic number verification [FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer [FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer } /// /// BucketNode操作的静态方法集合 /// private static class BucketNodeOperations { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNullOrEmpty(BucketNode* node) { return node == null || node->Magic != BucketNode.MAGIC_NUMBER; } #region Lock Operations // 前驱锁操作(控制PreviousLevel字段) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryAcquirePrevLock(BucketNode* node) { if (node == null) { return true; } int thread = Environment.CurrentManagedThreadId; return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void AcquirePrevLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->PrevLock) != thread) { while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) { Thread.Sleep(0); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ReleasePrevLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->PrevLock) == thread) { Volatile.Write(ref node->PrevLock, 0); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsPrevLockHeld(BucketNode* node) => node != null && Volatile.Read(ref node->PrevLock) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsPrevLockHeldByCurrentThread(BucketNode* node) => node != null && Volatile.Read(ref node->PrevLock) == Environment.CurrentManagedThreadId; // 后继锁操作(控制NextLevel字段) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryAcquireNextLock(BucketNode* node) { if (node == null) { return true; } int thread = Environment.CurrentManagedThreadId; return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void AcquireNextLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->NextLock) != thread) { while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) { Thread.Sleep(0); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ReleaseNextLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->NextLock) == thread) { Volatile.Write(ref node->NextLock, 0); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNextLockHeld(BucketNode* node) => node != null && Volatile.Read(ref node->NextLock) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNextLockHeldByCurrentThread(BucketNode* node) => node != null && Volatile.Read(ref node->NextLock) == Environment.CurrentManagedThreadId; // 双锁组合操作 [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool TryAcquireBothLocks(BucketNode* node) { if (node == null) { return true; } if (TryAcquirePrevLock(node)) { if (TryAcquireNextLock(node)) { return true; } else { ReleasePrevLock(node); // 回滚 return false; } } return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void ReleaseBothLocks(BucketNode* node) { if (node == null) { return; } ReleasePrevLock(node); ReleaseNextLock(node); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void AcquireBothLocks(BucketNode* node) { if (node == null) { return; } AcquirePrevLock(node); AcquireNextLock(node); } #endregion } } }