//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="HLSFAllocator.BucketNode.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the “Software”), to deal in the Software without restriction, // including without limitation the rights to use, copy, modify, merge, // publish, distribute, sublicense, and/or sell copies of the Software, // and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and // this permission notice shall be included in all copies // or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED “AS IS”, // WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, // INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. // IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, // ARISING FROM, OUT OF OR IN CONNECTION WITH // THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; namespace StgSharp.HighPerformance.Memory { public unsafe partial class HybridLayerSegregatedFitAllocator { private static void AcquireBucketNodeSpinLock(BucketNode* node) { // 使用HLSFAllocator的private static方法 AcquireBucketNodeBothLocks(node); } /// /// Initialize bucket system (called in constructor) /// private void InitializeBucketSystem() { _bucketHeads = new ulong[levelSize.Length * 3]; } private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node) { if (node == null || node->EntryRef == null) { return; } int index = (3 * levelIndex) + sizeIndex; if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle) { node->IsInBucket = true; node->Magic = BucketNode.MAGIC_NUMBER; return; } else { while (true) { // 对新节点只需要获取NextLock,因为我们要设置它的NextLevel AcquireBucketNodeNextLock(node); ulong head = Volatile.Read(ref _bucketHeads[index]); node->NextLevel = head; BucketNode* headNode = (BucketNode*)head; try { if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head) { // 对头节点只需要获取PrevLock,因为我们要设置它的PreviousLevel if (headNode != null && !TryAcquireBucketNodePrevLock(headNode)) { // reset state and retry _ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node); Thread.Sleep(0); continue; } try { node->IsInBucket = true; node->Magic = BucketNode.MAGIC_NUMBER; if (headNode != null) { headNode->PreviousLevel = (ulong)node; } return; } finally { if (headNode != null) { ReleaseBucketNodePrevLock(headNode); } } } else { Thread.Sleep(0); continue; } } finally { ReleaseBucketNodeNextLock(node); } } } } /// /// 根据锁类型释放BucketNode锁 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType) { switch (lockType) { case 0: ReleaseBucketNodePrevLock(node); break; // PrevLock case 1: ReleaseBucketNodeBothLocks(node); break; // BothLocks case 2: ReleaseBucketNodeNextLock(node); break; // NextLock } } private static void ReleaseBucketNodeSpinLock(BucketNode* node) { // 使用HLSFAllocator的private static方法 ReleaseBucketNodeBothLocks(node); } /// /// 按地址顺序释放BucketNode锁 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next) { // 直接释放,顺序不重要 if (previous != null) { ReleaseBucketNodeNextLock(previous); } if (node != null) { ReleaseBucketNodeBothLocks(node); } if (next != null) { ReleaseBucketNodePrevLock(next); } } private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node) { int index = (3 * levelIndex) + sizeIndex; while (true) { // 检查是否是栈顶节点 if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node) { // 栈顶节点直接使用TryPopLevel if (TryPopLevel(levelIndex, sizeIndex, out _)) { return; } continue; } // 中间或底部节点的删除操作 // 获取当前节点的前后指针(无锁读取) ulong p = Volatile.Read(ref node->PreviousLevel); ulong n = Volatile.Read(ref node->NextLevel); BucketNode* previous = (BucketNode*)p; BucketNode* next = (BucketNode*)n; // 直接抢锁策略:按地址顺序获取锁,避免死锁 if (!TryAcquireOrderedBucketLocks(previous, node, next)) { Thread.Sleep(0); continue; } try { // 验证链表结构仍然有效(防止ABA问题) if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n) { Thread.Sleep(0); continue; } // 执行链表操作 if (previous != null) { previous->NextLevel = n; } if (next != null) { next->PreviousLevel = p; } node->NextLevel = EmptyHandle; node->PreviousLevel = EmptyHandle; node->IsInBucket = false; return; } finally { ReleaseOrderedBucketLocks(previous, node, next); } } } private static bool TryAcquireBucketNodeSpinLock(BucketNode* node) { // 使用HLSFAllocator的private static方法 return TryAcquireBucketNodeBothLocks(node); } /// /// 按地址顺序获取BucketNode锁,避免死锁 /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next) { // 创建排序数组(使用nuint数组存储地址和锁类型) nuint prevAddr = (nuint)previous; nuint nodeAddr = (nuint)node; nuint nextAddr = (nuint)next; // 使用简单的比较来确定锁的顺序 if (previous != null && node != null && next != null) { // 三个节点都存在,按地址排序 if (prevAddr < nodeAddr && nodeAddr < nextAddr) { // previous -> node -> next return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next); } else if (prevAddr < nextAddr && nextAddr < nodeAddr) { // previous -> next -> node return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node); } else if (nodeAddr < prevAddr && prevAddr < nextAddr) { // node -> previous -> next return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodePrevLock(next); } else if (nodeAddr < nextAddr && nextAddr < prevAddr) { // node -> next -> previous return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeNextLock(previous); } else if (nextAddr < prevAddr && prevAddr < nodeAddr) { // next -> previous -> node return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node); } else // nextAddr < nodeAddr && nodeAddr < prevAddr { // next -> node -> previous return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous); } } else if (previous != null && node != null) { // 只有previous和node if (prevAddr < nodeAddr) { return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node); } else { return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodeNextLock(previous); } } else if (node != null && next != null) { // 只有node和next if (nodeAddr < nextAddr) { return TryAcquireBucketNodeBothLocks(node) && TryAcquireBucketNodePrevLock(next); } else { return TryAcquireBucketNodePrevLock(next) && TryAcquireBucketNodeBothLocks(node); } } else if (node != null) { // 只有node return TryAcquireBucketNodeBothLocks(node); } return true; // 没有节点需要锁定 } private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node) { int index = (3 * levelIndex) + sizeIndex; while (true) { ulong h = Volatile.Read(ref _bucketHeads[index]); BucketNode* head = (BucketNode*)h; if (head == null) { node = null; return false; } // 对头节点获取NextLock,因为我们要读取它的NextLevel if (!TryAcquireBucketNodeNextLock(head)) { Thread.Sleep(0); continue; } ulong n = Volatile.Read(ref head->NextLevel); BucketNode* next = (BucketNode*)n; try { // 对next节点获取PrevLock,因为我们要修改它的PreviousLevel if (next != null && !TryAcquireBucketNodePrevLock(next)) { Thread.Sleep(0); continue; } try { if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h) { node = head; node->IsInBucket = false; node->NextLevel = EmptyHandle; if (next != null) { next->PreviousLevel = EmptyHandle; } return true; } else { Thread.Sleep(0); continue; } } finally { if (next != null) { ReleaseBucketNodePrevLock(next); } } } finally { ReleaseBucketNodeNextLock(head); } } } /// /// BucketNode structure - specialized node for bucket linked lists Stored in main /// allocation area (user-overwritable region) /// [StructLayout(LayoutKind.Explicit, Size = 40)] internal struct BucketNode { public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification [FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry [FieldOffset(28)] internal bool IsInBucket; // Whether in bucket [FieldOffset(36)] internal int NextLock; // 后继指针锁(控制NextLevel指针操作) // 双锁设计:分离前后指针操作 [FieldOffset(32)] internal int PrevLock; // 前驱指针锁(控制PreviousLevel指针操作) [FieldOffset(24)] internal uint Magic; // Magic number verification [FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer [FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer } #region BucketNode Operations [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsNullOrEmpty(BucketNode* node) { return node == null || node->Magic != BucketNode.MAGIC_NUMBER; } // BucketNode 前驱锁操作(控制PreviousLevel字段) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryAcquireBucketNodePrevLock(BucketNode* node) { if (node == null) { return true; } int thread = Environment.CurrentManagedThreadId; return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void AcquireBucketNodePrevLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->PrevLock) != thread) { while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) { Thread.Sleep(0); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReleaseBucketNodePrevLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->PrevLock) == thread) { Volatile.Write(ref node->PrevLock, 0); } } // BucketNode 后继锁操作(控制NextLevel字段) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryAcquireBucketNodeNextLock(BucketNode* node) { if (node == null) { return true; } int thread = Environment.CurrentManagedThreadId; return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void AcquireBucketNodeNextLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->NextLock) != thread) { while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) { Thread.Sleep(0); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReleaseBucketNodeNextLock(BucketNode* node) { if (node == null) { return; } int thread = Environment.CurrentManagedThreadId; if (Volatile.Read(ref node->NextLock) == thread) { Volatile.Write(ref node->NextLock, 0); } } // BucketNode 双锁组合操作 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryAcquireBucketNodeBothLocks(BucketNode* node) { if (node == null) { return true; } if (TryAcquireBucketNodePrevLock(node)) { if (TryAcquireBucketNodeNextLock(node)) { return true; } else { ReleaseBucketNodePrevLock(node); // 回滚 return false; } } return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReleaseBucketNodeBothLocks(BucketNode* node) { if (node == null) { return; } ReleaseBucketNodePrevLock(node); ReleaseBucketNodeNextLock(node); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void AcquireBucketNodeBothLocks(BucketNode* node) { if (node == null) { return; } AcquireBucketNodePrevLock(node); AcquireBucketNodeNextLock(node); } #endregion } }