add from giga-device, new branch added
This commit is contained in:
Nitload Youmu
+143
@@ -0,0 +1,143 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevel(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevel(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1]; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->BlockSize = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevel(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevel(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1]; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->BlockSize = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, Environment.CurrentManagedThreadId, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and level
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
if (level < 0 || level >= levelSize.Length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int segmentIndex = (int)size / levelSize[level];
|
||||
return Math.Min(2, segmentIndex); // Ensure not exceeding maximum segment index
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->BlockSize + other->BlockSize;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->BlockSize = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, Environment.CurrentManagedThreadId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry != null &&
|
||||
(Volatile.Read(ref entry->SpinLock) == thread ||
|
||||
Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) == 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks
|
||||
/// </summary>
|
||||
private Entry* TryMergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint originalSize = entry->BlockSize;
|
||||
int originalLevel = GetLevelFromSize(originalSize);
|
||||
int originalSegment = DetermineSegmentIndex(originalSize, originalLevel);
|
||||
|
||||
// Forward merge: check predecessor nodes in position linked list
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
while (prev != null && prev != _spareMemory && prev->State == EntryState.Empty && result->IsAdjacentTo(prev))
|
||||
{
|
||||
// Try acquire spin lock for prev entry
|
||||
if (!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (prev->State != EntryState.Empty || !result->IsAdjacentTo(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate prev's bucket position
|
||||
uint prevSize = prev->BlockSize;
|
||||
int prevLevel = GetLevelFromSize(prevSize);
|
||||
int prevSegment = DetermineSegmentIndex(prevSize, prevLevel);
|
||||
|
||||
// Remove prev node from bucket (O(1) operation!)
|
||||
BucketNode* prevBucketNode = ValidateOrRepairBucketNode(prev);
|
||||
if (prevBucketNode != null && RemoveFromBucket(prevLevel, prevSegment, prevBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(prev);
|
||||
|
||||
// Remove prev from position linked list
|
||||
RemoveFromPositionChain(prev);
|
||||
|
||||
// Free prev's Entry node
|
||||
_nodes.Free((nuint)prev);
|
||||
|
||||
// Continue searching forward
|
||||
prev = (Entry*)result->PreviousNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
// Backward merge: check successor nodes in position linked list
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
while (next != null && next->State == EntryState.Empty && result->IsAdjacentTo(next))
|
||||
{
|
||||
// Try acquire spin lock for next entry
|
||||
if (!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (next->State != EntryState.Empty || !result->IsAdjacentTo(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate next's bucket position
|
||||
uint nextSize = next->BlockSize;
|
||||
int nextLevel = GetLevelFromSize(nextSize);
|
||||
int nextSegment = DetermineSegmentIndex(nextSize, nextLevel);
|
||||
|
||||
// Remove next node from bucket (O(1) operation!)
|
||||
BucketNode* nextBucketNode = ValidateOrRepairBucketNode(next);
|
||||
if (nextBucketNode != null && RemoveFromBucket(nextLevel, nextSegment, nextBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(next);
|
||||
|
||||
// Remove next from position linked list
|
||||
RemoveFromPositionChain(next);
|
||||
|
||||
// Free next's Entry node
|
||||
_nodes.Free((nuint)next);
|
||||
|
||||
// Continue searching backward
|
||||
next = (Entry*)result->NextNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint BlockSize; // Block size information
|
||||
[FieldOffset(36)] internal uint Reverse;
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly bool IsAdjacentTo(Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->BlockSize == this.Position ||
|
||||
this.Position + this.BlockSize == other->Position;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Level.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private const int MaxLevel = 9;
|
||||
|
||||
private static readonly int[] levelSize = [
|
||||
64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216
|
||||
];
|
||||
|
||||
public int AlignOfLevel(int bucketLevel)
|
||||
{
|
||||
int blockSize = levelSize[bucketLevel];
|
||||
return int.Min(blockSize, _align);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetLevelFromSize(uint size)
|
||||
{
|
||||
/*
|
||||
rebuild by using binary operations
|
||||
*/
|
||||
|
||||
|
||||
// Ensure minimum size
|
||||
if (size <= 64)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Find the appropriate level for the given size
|
||||
for (int i = 0; i < levelSize.Length - 1; i++)
|
||||
{
|
||||
if (size <= levelSize[i]) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// If size is larger than all predefined levels, use the last level
|
||||
return levelSize.Length - 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public enum CommonMemoryAlign : int
|
||||
{
|
||||
|
||||
XMMLevel = 16,
|
||||
YMMLevel = 32,
|
||||
ZMMLevel = 64,
|
||||
|
||||
ArmNEONLevel = 16,
|
||||
|
||||
}
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private BufferExpansionLock _spareLock = new();
|
||||
|
||||
public hlsfHandle Alloc(uint size)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan<nuint>(size, (nuint)levelSize[^1] * 2);
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
if (TryPopLevelForAllocation(i, s, out Entry* handle))
|
||||
{
|
||||
handle->State = EntryState.Allocated;
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i >= levelSize.Length)
|
||||
{
|
||||
if (TryAllocateNew16MBBlock(out handle))
|
||||
{
|
||||
break;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSize[level]);
|
||||
handle->State = EntryState.Allocated;
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset) { }
|
||||
|
||||
/// <summary>
|
||||
/// Try to allocate a new 16MB block when all buckets are empty
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Output handle to the allocated Entry
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSize[^1]; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
|
||||
// Check if we have enough space
|
||||
if (newPos > ((nuint)m_Buffer) + _size)
|
||||
{
|
||||
handle = null;
|
||||
return false; // Out of memory
|
||||
}
|
||||
|
||||
// Try to allocate new Entry from slab allocator
|
||||
try
|
||||
{
|
||||
handle = (Entry*)_nodes.Allocate();
|
||||
handle->Position = pos;
|
||||
handle->BlockSize = (uint)blockSize;
|
||||
handle->State = EntryState.ThreadOccupied;
|
||||
handle->NextNear = EmptyHandle;
|
||||
handle->PreviousNear = EmptyHandle;
|
||||
handle->SpinLock = 0;
|
||||
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
entry = bucketNode->EntryRef;
|
||||
return entry != null;
|
||||
}
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and level
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
if (level < 0 || level >= levelSize.Length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int segmentIndex = (int)size / levelSize[level];
|
||||
return Math.Min(2, segmentIndex); // Ensure not exceeding maximum segment index
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->BlockSize + other->BlockSize;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->BlockSize = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (entry != null && Volatile.Read(ref entry->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry == null ||
|
||||
(Interlocked.CompareExchange(ref entry->SpinLock, thread,
|
||||
0) == 0 ||
|
||||
Volatile.Read(ref entry->SpinLock) == thread);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks
|
||||
/// </summary>
|
||||
private Entry* TryMergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint originalSize = entry->BlockSize;
|
||||
int originalLevel = GetLevelFromSize(originalSize);
|
||||
int originalSegment = DetermineSegmentIndex(originalSize, originalLevel);
|
||||
|
||||
// Forward merge: check predecessor nodes in position linked list
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
while (prev != null && prev != _spareMemory && prev->State == EntryState.Empty && result->IsAdjacentTo(prev))
|
||||
{
|
||||
// Try acquire spin lock for prev entry
|
||||
if (!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (prev->State != EntryState.Empty || !result->IsAdjacentTo(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate prev's bucket position
|
||||
uint prevSize = prev->BlockSize;
|
||||
int prevLevel = GetLevelFromSize(prevSize);
|
||||
int prevSegment = DetermineSegmentIndex(prevSize, prevLevel);
|
||||
|
||||
// Remove prev node from bucket (O(1) operation!)
|
||||
BucketNode* prevBucketNode = ValidateOrRepairBucketNode(prev);
|
||||
if (prevBucketNode != null && RemoveFromBucket(prevLevel, prevSegment, prevBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(prev);
|
||||
|
||||
// Remove prev from position linked list
|
||||
RemoveFromPositionChain(prev);
|
||||
|
||||
// Free prev's Entry node
|
||||
_nodes.Free((nuint)prev);
|
||||
|
||||
// Continue searching forward
|
||||
prev = (Entry*)result->PreviousNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
// Backward merge: check successor nodes in position linked list
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
while (next != null && next->State == EntryState.Empty && result->IsAdjacentTo(next))
|
||||
{
|
||||
// Try acquire spin lock for next entry
|
||||
if (!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (next->State != EntryState.Empty || !result->IsAdjacentTo(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate next's bucket position
|
||||
uint nextSize = next->BlockSize;
|
||||
int nextLevel = GetLevelFromSize(nextSize);
|
||||
int nextSegment = DetermineSegmentIndex(nextSize, nextLevel);
|
||||
|
||||
// Remove next node from bucket (O(1) operation!)
|
||||
BucketNode* nextBucketNode = ValidateOrRepairBucketNode(next);
|
||||
if (nextBucketNode != null && RemoveFromBucket(nextLevel, nextSegment, nextBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(next);
|
||||
|
||||
// Remove next from position linked list
|
||||
RemoveFromPositionChain(next);
|
||||
|
||||
// Free next's Entry node
|
||||
_nodes.Free((nuint)next);
|
||||
|
||||
// Continue searching backward
|
||||
next = (Entry*)result->NextNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint BlockSize; // Block size information
|
||||
[FieldOffset(36)] internal uint Reverse;
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly bool IsAdjacentTo(Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->BlockSize == this.Position ||
|
||||
this.Position + this.BlockSize == other->Position;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return node == null ||
|
||||
(Interlocked.CompareExchange(ref node->SpinLock, thread,
|
||||
0) == 0 ||
|
||||
Volatile.Read(ref node->SpinLock) == thread);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
while (Interlocked.CompareExchange(ref entry->SpinLock, Environment.CurrentManagedThreadId, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and level
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int DetermineSegmentIndex(uint size, int level)
|
||||
{
|
||||
if (level < 0 || level >= levelSize.Length) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int segmentIndex = (int)size / levelSize[level];
|
||||
return Math.Min(2, segmentIndex); // Ensure not exceeding maximum segment index
|
||||
}
|
||||
|
||||
private void MergeEntry(Entry* remain, Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine merged starting position and size
|
||||
nuint newPosition = remain->Position < other->Position ? other->Position : other->Position;
|
||||
uint newSize = remain->BlockSize + other->BlockSize;
|
||||
|
||||
// Update current Entry
|
||||
remain->Position = newPosition;
|
||||
remain->BlockSize = newSize;
|
||||
}
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
entry-> PreviousNear = EmptyHandle;
|
||||
entry-> Position = EmptyHandle;
|
||||
_nodes.Free((nuint)entry);
|
||||
}
|
||||
|
||||
private void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
_ = Interlocked.CompareExchange(ref entry->SpinLock, 0, Environment.CurrentManagedThreadId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return entry != null &&
|
||||
(Volatile.Read(ref entry->SpinLock) == thread ||
|
||||
Interlocked.CompareExchange(ref entry->SpinLock, thread, 0) == 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks
|
||||
/// </summary>
|
||||
private Entry* TryMergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint originalSize = entry->BlockSize;
|
||||
int originalLevel = GetLevelFromSize(originalSize);
|
||||
int originalSegment = DetermineSegmentIndex(originalSize, originalLevel);
|
||||
|
||||
// Forward merge: check predecessor nodes in position linked list
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
while (prev != null && prev != _spareMemory && prev->State == EntryState.Empty && result->IsAdjacentTo(prev))
|
||||
{
|
||||
// Try acquire spin lock for prev entry
|
||||
if (!TryAcquireEntrySpinLock(prev))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (prev->State != EntryState.Empty || !result->IsAdjacentTo(prev))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate prev's bucket position
|
||||
uint prevSize = prev->BlockSize;
|
||||
int prevLevel = GetLevelFromSize(prevSize);
|
||||
int prevSegment = DetermineSegmentIndex(prevSize, prevLevel);
|
||||
|
||||
// Remove prev node from bucket (O(1) operation!)
|
||||
BucketNode* prevBucketNode = ValidateOrRepairBucketNode(prev);
|
||||
if (prevBucketNode != null && RemoveFromBucket(prevLevel, prevSegment, prevBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(prev);
|
||||
|
||||
// Remove prev from position linked list
|
||||
RemoveFromPositionChain(prev);
|
||||
|
||||
// Free prev's Entry node
|
||||
_nodes.Free((nuint)prev);
|
||||
|
||||
// Continue searching forward
|
||||
prev = (Entry*)result->PreviousNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(prev);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
// Backward merge: check successor nodes in position linked list
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
while (next != null && next->State == EntryState.Empty && result->IsAdjacentTo(next))
|
||||
{
|
||||
// Try acquire spin lock for next entry
|
||||
if (!TryAcquireEntrySpinLock(next))
|
||||
{
|
||||
break; // Cannot acquire lock, might be operated by other thread
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Double-check state after acquiring lock
|
||||
if (next->State != EntryState.Empty || !result->IsAdjacentTo(next))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate next's bucket position
|
||||
uint nextSize = next->BlockSize;
|
||||
int nextLevel = GetLevelFromSize(nextSize);
|
||||
int nextSegment = DetermineSegmentIndex(nextSize, nextLevel);
|
||||
|
||||
// Remove next node from bucket (O(1) operation!)
|
||||
BucketNode* nextBucketNode = ValidateOrRepairBucketNode(next);
|
||||
if (nextBucketNode != null && RemoveFromBucket(nextLevel, nextSegment, nextBucketNode))
|
||||
{
|
||||
// Merge into result
|
||||
result->MergeWith(next);
|
||||
|
||||
// Remove next from position linked list
|
||||
RemoveFromPositionChain(next);
|
||||
|
||||
// Free next's Entry node
|
||||
_nodes.Free((nuint)next);
|
||||
|
||||
// Continue searching backward
|
||||
next = (Entry*)result->NextNear;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(next);
|
||||
}
|
||||
|
||||
break; // Cannot remove, might be operated by other thread
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Entry structure - streamlined version (only position and state information) Stored in
|
||||
/// external SLAB
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct Entry
|
||||
{
|
||||
|
||||
[FieldOffset(24)] internal int SpinLock; // Spin lock for thread safety protection
|
||||
[FieldOffset(28)] internal int State; // Node state
|
||||
[FieldOffset(32)] internal uint BlockSize; // Block size information
|
||||
[FieldOffset(36)] internal uint Reverse;
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(0)] public nuint Position; // Memory block position
|
||||
|
||||
/// <summary>
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly bool IsAdjacentTo(Entry* other)
|
||||
{
|
||||
if (other == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check if this immediately follows other
|
||||
return other->Position + other->BlockSize == this.Position ||
|
||||
this.Position + this.BlockSize == other->Position;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(ulong index)
|
||||
{
|
||||
return index == EmptyHandle;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmptyIndex(Entry* entry)
|
||||
{
|
||||
return entry == null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merge two adjacent Entries (caller must ensure they are adjacent and both Empty
|
||||
/// state)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void MergeWith(Entry* other) { }
|
||||
|
||||
}
|
||||
|
||||
private static class EntryState
|
||||
{
|
||||
|
||||
public const int Allocated = 1; // Already allocated to user
|
||||
public const int Empty = 0; // Free, can be allocated
|
||||
public const int ThreadOccupied = 2; // Occupied by thread (being operated on)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return node != null &&
|
||||
(Volatile.Read(ref node->SpinLock) == thread ||
|
||||
Interlocked.CompareExchange(ref node->SpinLock, thread, 0) == 0);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->SpinLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
private void InitializeBucketSystem()
|
||||
{
|
||||
_bucketHeads = new ulong[levelSize.Length * 3];
|
||||
}
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (!TryAcquireBucketNodeSpinLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(headNode);
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReleaseBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (node != null && Volatile.Read(ref node->SpinLock) != 0) {
|
||||
_ = Interlocked.CompareExchange(ref node->SpinLock, 0, thread);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
// top of stack
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node && TryAcquireBucketNodeSpinLock(node))
|
||||
{
|
||||
_ = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
} else
|
||||
{
|
||||
AcquireBucketNodeSpinLock(node);
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel), n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p, next = (BucketNode*)n;
|
||||
try
|
||||
{
|
||||
if (next is null)
|
||||
{
|
||||
// bottom of stack
|
||||
if (!TryAcquireBucketNodeSpinLock(previous))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if (!(TryAcquireBucketNodeSpinLock(previous) && TryAcquireBucketNodeSpinLock(next)))
|
||||
{
|
||||
// middle of stack
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
previous ->NextLevel = n;
|
||||
next->PreviousLevel = p;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(node);
|
||||
ReleaseBucketNodeSpinLock(previous);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return node != null &&
|
||||
(Volatile.Read(ref node->SpinLock) == thread ||
|
||||
Interlocked.CompareExchange(ref node->SpinLock, thread, 0) == 0);
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
while (true)
|
||||
{
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
BucketNode* next = null;
|
||||
try
|
||||
{
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
if (!TryAcquireBucketNodeSpinLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
next = (BucketNode*)n;
|
||||
if (!TryAcquireBucketNodeSpinLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeSpinLock(head);
|
||||
ReleaseBucketNodeSpinLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 36)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
public const uint MAGIC_NUMBER = 0xBEEFCAFE; // Magic number for node integrity verification
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(32)] internal int SpinLock; // Spin lock for bucket head operation
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user