add from giga-device, new branch added
This commit is contained in:
Nitload Youmu
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Linq.Expressions;
|
||||
using System.Reflection.PortableExecutable;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private bool AssertAlignBoundry(ulong current, Entry* target, int level)
|
||||
{
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == target->Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
return (int)size / levelSize[level];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsAdjacentTo(Entry* current, Entry* other)
|
||||
{
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->Size == current->Position ||
|
||||
current->Position + current->Size == other->Position;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// stretagey, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = -1;
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
if (TryAcquireEntrySpinLock(entry)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// current Level merge
|
||||
while (originLevel != entry->Level)
|
||||
{
|
||||
ulong offset = entry->Position - (ulong)m_Buffer;
|
||||
AcquireBucketNodeSpinLock((BucketNode*)entry->Position);
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
RemoveFromBucket(originLevel, , (BucketNode*)near->Position);
|
||||
}
|
||||
|
||||
// forward merging
|
||||
|
||||
while (true)
|
||||
{
|
||||
ulong n = entry->NextNear;
|
||||
Entry* near = (Entry*)n;
|
||||
if (near == null ||
|
||||
near->State != EntryState.Empty ||
|
||||
!IsAdjacentTo(entry, near) ||
|
||||
entry->Level != near->Level ||
|
||||
!AssertAlignBoundry(offset, near, entry->Level) ||
|
||||
!TryAcquireEntrySpinLock(near))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->Size + other->Size;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->Size = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(36)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint Size; // Block size information
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
e->State = EntryState.ThreadOccupied;
|
||||
|
||||
// Step 4: Try merge with adjacent free blocks and remove them from buckets
|
||||
Entry* mergedEntry = TryMergeAndRemoveFromBuckets(e);
|
||||
|
||||
|
||||
// Step 5: Calculate final level and segment for the merged block
|
||||
uint finalSize = mergedEntry->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
|
||||
|
||||
// Step 6: Set the merged entry state to Empty (ready to be reused)
|
||||
mergedEntry->State = EntryState.Empty;
|
||||
|
||||
// Step 7: Push the merged entry back to appropriate bucket
|
||||
PushLevel(finalLevel, finalSegment, (BucketNode*)mergedEntry->Position);
|
||||
|
||||
// Note: If mergedEntry is different from original e, the original e
|
||||
// and any merged entries have already been freed to _nodes by TryMergeAndRemoveFromBuckets
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Step 8: Always release the spin lock
|
||||
ReleaseEntrySpinLock(e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+356
@@ -0,0 +1,356 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation (保留用于兼容)
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// 字节级原子操作 - 使用lock-free的自旋方式
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
byte current;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref StateMask);
|
||||
if (current != expectedState) {
|
||||
return false;
|
||||
}
|
||||
} while (Interlocked.CompareExchange(ref StateMask, newState, expectedState) != expectedState);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref LeftOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref RightOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref SelfOpMask, OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private static readonly nuint EmptyHandle = (nuint)(void*)null;
|
||||
|
||||
// Bucket system field - directly in main allocator class
|
||||
private ulong[] _bucketHeads;
|
||||
|
||||
private Entry* _spareMemory;
|
||||
private readonly byte* m_Buffer;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _nodes;
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
{
|
||||
if (byteSize < 64) {
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
_size = byteSize;
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
|
||||
_align = 16;
|
||||
|
||||
|
||||
// 初始化桶系统
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 8 bytes.");
|
||||
}
|
||||
_nodes = SlabAllocator<Entry>.Create(64, BufferLayout.Chunked);
|
||||
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
|
||||
InitializeBucketSystem();
|
||||
|
||||
|
||||
// 初始化spareMemory
|
||||
_spareMemory = (Entry*)_nodes.Allocate();
|
||||
_spareMemory->State = EntryState.ThreadOccupied;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_nodes.Dispose();
|
||||
}
|
||||
NativeMemory.AlignedFree(m_Buffer);
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
~HybridLayerSegregatedFitAllocator()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->TrySetState(BucketNode.StateEmpty, BucketNode.StateInBucket);
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return (node == null) ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0) ||
|
||||
(Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->TrySetState(BucketNode.StateInBucket, BucketNode.StateEmpty);
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(29)] internal byte LeftOpMask; // Left pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(30)] internal byte RightOpMask; // Right pointer operation (Free=0, Occupied=1)
|
||||
[FieldOffset(31)] internal byte SelfOpMask; // Self operation (Free=0, Occupied=1)
|
||||
|
||||
// Four independent byte masks for atomic operations
|
||||
[FieldOffset(28)] internal byte StateMask; // Usage state (Empty=0, InBucket=1, MarkedForRemoval=2)
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#region Four Independent Byte Masks
|
||||
|
||||
// State constants
|
||||
public const byte StateEmpty = 0;
|
||||
public const byte StateInBucket = 1;
|
||||
public const byte StateMarkedForRemoval = 2;
|
||||
public const byte StateReserved = 3;
|
||||
|
||||
// Operation constants
|
||||
public const byte OpFree = 0;
|
||||
public const byte OpOccupied = 1;
|
||||
|
||||
// Atomic state operations
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TrySetState(byte expectedState, byte newState)
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref StateMask), newState, expectedState) == expectedState;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public byte GetState() => Volatile.Read(ref StateMask);
|
||||
|
||||
// Left operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireLeftOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref LeftOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseLeftOp()
|
||||
{
|
||||
Volatile.Write(ref LeftOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsLeftOpOccupied() => Volatile.Read(ref LeftOpMask) == OpOccupied;
|
||||
|
||||
// Right operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireRightOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref RightOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseRightOp()
|
||||
{
|
||||
Volatile.Write(ref RightOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsRightOpOccupied() => Volatile.Read(ref RightOpMask) == OpOccupied;
|
||||
|
||||
// Self operation atomic methods
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryAcquireSelfOp()
|
||||
{
|
||||
return Interlocked.CompareExchange(ref Unsafe.As<byte, int>(ref SelfOpMask), OpOccupied, OpFree) == OpFree;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ReleaseSelfOp()
|
||||
{
|
||||
Volatile.Write(ref SelfOpMask, OpFree);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsSelfOpOccupied() => Volatile.Read(ref SelfOpMask) == OpOccupied;
|
||||
|
||||
// Convenience properties
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsInBucket() => GetState() == StateInBucket;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsEmpty() => GetState() == StateEmpty;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMarkedForRemoval() => GetState() == StateMarkedForRemoval;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool HasAnyOperation() => IsLeftOpOccupied() || IsRightOpOccupied() || IsSelfOpOccupied();
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user