youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MulAvailable

Browse Source
This commit is contained in:
Nitload
2025-12-16 23:13:17 +08:00
parent 28ea5d1d1e
commit 3486375470
2637 changed files with 23787 additions and 838773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/BufferStack.cs
+163
View File
@@ -0,0 +1,163 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="BufferStack"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
public unsafe class BufferStack<T> : IDisposable where T : unmanaged
{
private byte* _buffer;
private ulong _count;
public BufferStack(int capacity)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
AllocateBuffer(capacity);
}
public int Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (int)_count;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
((Header*)_buffer)->Top = 0;
_count = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose()
{
Marshal.FreeHGlobal((nint)_buffer);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T Pop()
{
Header* header = (Header*)_buffer;
if (header->Top == 0) {
throw new InvalidOperationException("Stack underflow");
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
return dataStart[header->Top];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Push(T item)
{
Header* header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
Expand();
header = (Header*)_buffer;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[header->Top] = item;
header->Top++;
_count++;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPop(out T value)
{
Header* header = (Header*)_buffer;
if (header->Top == 0)
{
value = default;
return false;
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[header->Top];
return true;
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = (nuint)span.Length;
_count = (ulong)span.Length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = (nuint)capacity;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void Expand()
{
Header* header = (Header*)_buffer;
nuint newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
header = (Header*)_buffer;
header->Capacity = newCap;
}
private struct Header
{
public nuint Capacity;
public nuint Top;
}
}
}
StgSharp.Common/HighPerformance/Memory/ConcurrentBufferStack.cs
+309
View File
@@ -0,0 +1,309 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="ConcurrentBufferStack"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public static class BufferStackBuilder
{
// For BufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BufferStack<T> Create<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new BufferStack<T>(4); // Default small capacity
}
BufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
// For ConcurrentBufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ConcurrentBufferStack<T> CreateConcurrent<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new ConcurrentBufferStack<T>(4); // Default small capacity
}
ConcurrentBufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
}
[CollectionBuilder(builderType: typeof(BufferStackBuilder), methodName: nameof(BufferStackBuilder.CreateConcurrent))]
public unsafe class ConcurrentBufferStack<T> : IDisposable where T: unmanaged
{
private byte* _buffer;
private readonly BufferExpansionLock _lock = new();
private long _count;
public ConcurrentBufferStack()
{
AllocateBuffer(4);
}
public ConcurrentBufferStack(int capacity)
{
if (capacity <= 0) {
throw new ArgumentOutOfRangeException(nameof(capacity));
}
AllocateBuffer(capacity);
}
public ulong Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (ulong)Volatile.Read(ref _count);
}
public void Clear()
{
_lock.EnterExpansionProcess();
try
{
_lock.EnterBufferCopy(); // exclusive while mutating Top
try
{
Header* header = (Header*)_buffer;
header->Top = 0;
Volatile.Write(ref _count, 0);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
public void Dispose()
{
_lock.Dispose();
NativeMemory.Free(_buffer);
}
public T Pop()
{
while (true)
{
_lock.EnterBufferRead(); // prevent buffer being reallocated while reading/modifying Top
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0) {
throw new InvalidOperationException("Stack underflow");
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
_ = Interlocked.Decrement(ref _count);
return dataStart[oldTop - 1];
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
public void Push(T item)
{
while (true)
{
// Ensure capacity, expanding if needed
_lock.EnterMetaDataRead();
try
{
Header* header = (Header*)_buffer;
int topSnapshot = Volatile.Read(ref header->Top);
int cap = Volatile.Read(ref header->Capacity);
if (topSnapshot >= cap)
{
// Need expansion
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
int newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + ((nuint)newCap * (nuint)sizeof(T));
_lock.EnterBufferCopy();
try
{
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
((Header*)_buffer)->Capacity = newCap;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
finally
{
_lock.ExitExpansionProcess();
}
continue;
}
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
// Reserve and write under exclusive buffer lock to avoid consumers reading unwritten data
_lock.EnterBufferCopy();
try
{
Header* header = (Header*)_buffer;
int idx = header->Top; // reserve next slot
if (idx >= header->Capacity)
{
// lost race against expander or other producer; retry
continue;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[idx] = item; // write first
header->Top = idx + 1; // then publish by increasing top
_ = Interlocked.Increment(ref _count);
return;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
public bool TryPop(out T value)
{
while (true)
{
_lock.EnterBufferRead();
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0)
{
value = default;
return false;
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[oldTop - 1];
_ = Interlocked.Decrement(ref _count);
return true;
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart,
(nuint)header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = span.Length;
Volatile.Write(ref _count, span.Length);
}
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + (nuint)capacity * (nuint)sizeof(T);
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = capacity;
}
private struct Header
{
public int Capacity; // max capacity of the stack (number of elements)
public int Top; // next available index in the stack
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs
+114
View File
@@ -0,0 +1,114 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Drawing;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Allocator = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public readonly unsafe struct HybridLayerSegregatedFitAllocationHandle
{
internal readonly Allocator.Entry* EntryHandle;
public readonly uint AllocSize;
internal HybridLayerSegregatedFitAllocationHandle(Allocator.Entry* handle, uint s)
{
EntryHandle = handle;
Pointer = (byte*)handle->Position;
AllocSize = s;
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
}
public readonly ref byte this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual<uint>(index, AllocSize);
return ref Pointer[index];
}
}
public readonly byte* Pointer { get; init; }
public readonly Span<byte> BufferHandle
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => new(Pointer, (int)AllocSize);
}
public readonly Enumerator GetEnumerator()
{
return new Enumerator(this);
}
public ref struct Enumerator : IEnumerator<byte>
{
private HybridLayerSegregatedFitAllocationHandle _handle;
private uint _index = ~0u;
internal Enumerator(HybridLayerSegregatedFitAllocationHandle span)
{
_handle = span;
Span = span.BufferHandle;
}
public ref byte RefCurrent => ref _handle[_index];
public Span<byte> Span { get; init; }
public readonly void Dispose() { }
public bool MoveNext()
{
_index++;
return _index < _handle.AllocSize;
}
public void Reset()
{
_index = 0;
}
object IEnumerator.Current => RefCurrent;
byte IEnumerator<byte>.Current => RefCurrent;
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+242
View File
@@ -0,0 +1,242 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.ComponentModel;
using System.Diagnostics.Tracing;
using System.Numerics;
using System.Runtime.InteropServices;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
public hlsfHandle Alloc(uint size)
{
if (size > 32U * 1024U * 1024U)
{
Entry* e = (Entry*)_entries.Allocate();
e->Size = size;
e->Position = (nuint)NativeMemory.AlignedAlloc(size, (nuint)_align);
e->State = EntryState.Large;
return new hlsfHandle(e, size);
}
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(size, level);
int i = level, s = segmentIndex;
Entry* handle;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out handle))
{
break;
} else if (TryAllocateNew32MBBlock(out handle))
{
break;
}
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
}
if (handle != null)
{
int requestedOffset = (segmentIndex + 1) * levelSizeArray[level];
if (requestedOffset > handle->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
long remainder = handle->Size - requestedOffset;
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
handle->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(handle->NextNear, remainder);
}
}
return new hlsfHandle(handle, size);
}
/// <summary>
/// Slice the memory block into smaller blocks and add them to the appropriate buckets
/// Notice: this method is not responsible for freeing the original BucketNode
/// </summary>
/// <param name="e">
///
/// </param>
/// <param name="sizeToSlice">
///
/// </param>
private void SliceMemory(Entry* e, long sizeToSlice)
{
Entry* next = e;
long remain = sizeToSlice; // remained size to slice
long offset = (long)e->Position - sizeToSlice - (long)m_Buffer; // offset of buffer to slice
long end = (long)e->Position - (long)m_Buffer; // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
i = int.Min(10, i);
for (; i < levelSizeArray.Length; i++)
{
int size = levelSizeArray[i];
int upper = size * 4;
long edge = (offset + upper - 1) / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
if (i > MaxLevel)
{
int size = levelSizeArray[^1];
int upper = size * 4;
long edge = end / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / upper);
if (count > 0)
{
long newSize = (long)count * (long)upper;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
if (remain <= 0) {
return;
}
}
}
for (i--; i >= 0; i--)
{
int size = levelSizeArray[i];
long edge = end / size * size;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
b->NextLevel = null;
b->PreviousLevel = null;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
}
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew32MBBlock(out Entry* handle)
{
nuint pos = _spareMemory->Position;
long remain = _spareMemory->Size;
if (remain <= 0)
{
handle = null;
return false;
}
long size = remain > 32 * 1024 * 1024L ? 32 * 1024 * 1024L : remain;
// build new entry
Entry* e = (Entry*)_entries.Allocate();
SetEntryLevel(e, pos, size);
InsertEntryBefore(e, _spareMemory);
e->State = EntryState.Empty;
// set fields of spare memory
_spareMemory->Size -= size;
_spareMemory->Position += (nuint)size;
handle = e;
return true;
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+171
View File
@@ -0,0 +1,171 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Reflection.Metadata;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private void PushBucketToLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
node->IsInBucket = true;
node->NextLevel = null;
node->PreviousLevel = null;
// Console.WriteLine($"Push to empty level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}");
return;
} else
{
BucketNode* head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = head;
node->IsInBucket = true;
_bucketHeads[index] = node;
// Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}");
if (headNode != null)
{
headNode->PreviousLevel = node;
}
return;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = (BucketNode*)null;
node->PreviousLevel = (BucketNode*)null;
node->IsInBucket = false;
return;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex;
BucketNode* head = _bucketHeads[index];
if (head == null)
{
node = null;
return false;
}
// Console.WriteLine($"level popped from level {levelIndex} {sizeIndex}, node is at {(ulong)head}");
BucketNode* next = head->NextLevel;
if (next == null)
{
_bucketHeads[index] = null;
node = head;
return true;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = null;
if (next != null) {
next->PreviousLevel = null;
}
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
return ok;
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = null;
node->PreviousLevel = null;
node->IsInBucket = false;
node->EntryRef = null;
return true;
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit)]
internal struct BucketNode
{
[FieldOffset(0)] internal Entry* EntryRef; // Pointer to corresponding Entry
// [FieldOffset(16)] internal ulong LockBuffer;
[FieldOffset(8)] internal BucketNode* NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(24)] internal BucketNode* PreviousLevel; // Bucket stack linked list: previous pointer
[FieldOffset(32)] internal bool IsInBucket; // Whether in bucket
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs
+123
View File
@@ -0,0 +1,123 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Debug"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public sealed class HLSFPositionChainBreakException : Exception
{
public HLSFPositionChainBreakException()
: base("Position chain of an HLSF allocator is broken") { }
}
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private object _lock = new object();
public void DisplayEntryLayout(bool singleStep)
{
lock (_lock)
{
Entry* cur = _spareMemory;
while (cur != null)
{
Console.WriteLine(EntryToString(cur));
cur = cur->PreviousNear;
if (singleStep) {
Console.ReadKey();
}
}
}
}
public void DumpPositionChain(bool singleStep = false, int maxCount = 4096)
{
lock (_lock)
{
try
{
Entry* cur = _spareMemory;
int cnt = 0;
if (cur != null) {
cur = cur->NextNear;
}
Console.WriteLine(EntryToString(_spareMemory));
while (cur != null && cur != _spareMemory && cnt < maxCount)
{
Console.WriteLine(EntryToString(cur));
cnt++;
if (singleStep) {
Console.ReadKey();
}
cur = cur->NextNear;
}
if (cur == _spareMemory)
{
Console.WriteLine("(reached sentinel)");
} else if (cnt >= maxCount) {
Console.WriteLine("(stopped at maxCount)");
}
}
catch (Exception ex)
{
Console.WriteLine($"[DumpPositionChain-forward] Exception: {ex.Message}");
}
}
}
public IEnumerable<string> ShowEntryLayout()
{
lock (_lock)
{
Stack<string> layout = [];
Entry* cur = _spareMemory;
while (cur != null)
{
layout.Push(EntryToString(cur));
cur = cur->PreviousNear;
}
return layout;
}
}
private static string EntryToString(Entry* e)
{
BucketNode* b = (BucketNode*)e->Position;
return $"{(ulong)e->PreviousNear}<-prev- Position:{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -next->{(ulong)e->NextNear}";
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+118
View File
@@ -0,0 +1,118 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Minimal collect: linear scan, merge contiguous empty Blocks into one, push merged head
/// back to bucket. Caller-responsible design: no slicing or redistribution.
/// </summary>
public void Collect()
{
Entry* cur = _spareMemory->NextNear;
long size = 0;
while (cur != null && cur != _spareMemory)
{
// cur is not empty, skip
if (cur->State != EntryState.Empty)
{
cur = cur->NextNear;
continue;
}
// remove cur from bucket
do
{
size += cur->Size;
Entry* next = cur->NextNear;
RemoveFromBucket(cur->Level, DetermineSegmentIndex(cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while (cur != null && cur != _spareMemory && cur->State == EntryState.Empty);
if (size > 0) {
SliceMemory(cur, size);
}
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// </param>
public void Free(hlsfHandle handle)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
if (e->State == EntryState.Large)
{
NativeMemory.AlignedFree((void*)e->Position);
_entries.Free((nuint)e);
return;
}
if (e->State != EntryState.Alloc) {
return;
}
// Mark as empty so it can be merged
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
long finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
// Console.WriteLine("one block freed");
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs
+103
View File
@@ -0,0 +1,103 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Level"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private const int MaxLevel = 9;
private static readonly int[] levelSizeArray = [
64,
256,
1024,
4096,
16384,
65536,
262144,
1048576,
4194304,
16777216
];
public int AlignOfLevel(int bucketLevel)
{
int blockSize = levelSizeArray[bucketLevel];
return int.Min(blockSize, _align);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(long size)
{
return GetLevelFromSize((uint)size);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(uint size)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216]
* Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level)
*
* For size S, we need: 2^(6+2*level) >= S
* So: 6+2*level >= log2(S)
* Therefore: level >= (log2(S) - 6) / 2
*
* We use hardware LZCNT instruction via BitOperations.LeadingZeroCount
* log2(S) = 31 - LeadingZeroCount(S) for 32-bit values
*/
int log2Size = 31 - BitOperations.LeadingZeroCount(size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return level;
}
}
public enum CommonMemoryAlign : int
{
XMMLevel = 16,
YMMLevel = 32,
ZMMLevel = 64,
ArmNEONLevel = 16,
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+208
View File
@@ -0,0 +1,208 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(Entry* current, Entry* other)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return other->Position + (nuint)other->Size == current->Position ||
current->Position + (nuint)current->Size == other->Position;
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
private bool AssertBoundary(Entry* previous, Entry* current, int level)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = levelSizeArray[level] * 4;
ulong prevHead = previous->Position - (ulong)m_Buffer,
currHead = current->Position - (ulong)m_Buffer;
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int DetermineSegmentIndex(long size, int level)
{
level = int.Min(level, MaxLevel);
return (int)((size - 1) / levelSizeArray[level]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryAfter(Entry* entry, Entry* position)
{
Entry* next = position->NextNear;
entry->NextNear = next;
entry->PreviousNear = position;
next->PreviousNear = entry;
position->NextNear = entry;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryBefore(Entry* entry, Entry* position)
{
Entry* prev = position->PreviousNear;
entry->NextNear = position;
entry->PreviousNear = prev;
prev->NextNear = entry;
position->PreviousNear = entry;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
long size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(size, originLevel);
do
{
// previous direction merge
Entry* p = entry->PreviousNear;
if (p == null ||
p == _spareMemory ||
p->State != EntryState.Empty ||
p->Level != entry->Level ||
!AssertBoundary(p, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
entry->Size += p->Size;
entry->Position = p->Position;
RemoveFromBucket(originLevel, DetermineSegmentIndex(p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
do
{
// next direction merge
Entry* n = entry->NextNear;
if (n == null ||
n == _spareMemory ||
n->State != EntryState.Empty ||
n->Level != entry->Level ||
!AssertBoundary(n, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
entry->Size += n->Size;
RemoveFromBucket(originLevel, DetermineSegmentIndex(n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += entry->Size >= levelSizeArray[originLevel] * 4 ? 1 : 0;
entry->Level &= 15;
BucketNode* bucket = (BucketNode*)entry->Position;
bucket->EntryRef = entry;
}
/// <summary>
/// Remove node from position linked list, this method does not recycle a BucketNode from
/// size bucket
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
Entry* prev = entry->PreviousNear;
Entry* next = entry->NextNear;
prev->NextNear = next;
next->PreviousNear = prev;
// Clear node pointers
entry->NextNear = null;
entry->PreviousNear = null;
// Console.WriteLine("entry recycled.");
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryLevel(Entry* entry, nuint position, long size)
{
entry ->Position = position;
entry->Size = size;
entry->Level = GetLevelFromSize(size);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal long Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(28)] public nuint Position; // Memory block position
}
internal enum EntryState : uint
{
Spare,
Empty,
Alloc,
Large
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
+115
View File
@@ -0,0 +1,115 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
// Bucket system field - directly in main allocator class
private readonly BucketNode*[] _bucketHeads;
// _spareMemory is also the beginning position of allocator
private Entry* _spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _entries;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
_size = byteSize;
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
_bucketHeads = new BucketNode*[levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry*)_entries.Allocate();
_spareMemory->State = EntryState.Spare;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
_bucketHeads = new BucketNode*[levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry*)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_entries.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md
+152
View File
@@ -0,0 +1,152 @@
# HLSF (Hybrid Layer Segregated Fit) Allocator
A constant-time (O(1)) allocator for real-time, high-throughput scenarios. It uses layered size classes with segmented buckets and stores all metadata externally for clean user memory and better cache locality.
## Key Points
- O(1) allocate/free with 10 levels: 64B x 16MB; each level split into 1x/2x/3x segments
- External metadata (`Entry`, `BucketNode`) in dedicated SLABs; user blocks have no headers
- Configurable alignment (>=16B); large block cap per allocation: 32MB
- Single-threaded implementation (external sync required for multi-threaded use)
## Quick Use
```csharp
using var hlsf = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024);
var h = hlsf.Alloc(1024);
Span<byte> s = h.BufferHandle; s[0] = 42;
hlsf.Free(h);
```
## Core APIs
### Alloc(uint size)
- Purpose: Allocate a block of at least `size` bytes.
- Input: `size` in bytes (<= 32MB for internal path; >32MB uses native large allocation).
- Output: `HybridLayerSegregatedFitAllocationHandle` with pointer access and `Span<byte>`.
- Behavior:
- Classify into level/segment and pop a matching bucket; escalate segments and levels if empty.
- Fallback to overflow bucket or carve 32MB from spare memory if all buckets are empty.
- Mark selected `Entry` as Alloc; slice any remainder into `Empty` entries and push to buckets.
- For >32MB requests, use `NativeMemory.AlignedAlloc` and mark `EntryState.Large`.
### Free(HybridLayerSegregatedFitAllocationHandle handle)
- Purpose: Free a previously allocated block.
- Behavior:
- Mark the corresponding `Entry` as `Empty`.
- Merge with adjacent `Empty` entries in both directions (boundary and level constraints).
- Remove merged neighbors from buckets, update position chain, and optionally promote level.
- Re-enqueue the final `Entry` into the appropriate bucket.
### Collect()
- Purpose: Reclaim and rebalance free entries without touching allocated blocks.
- Behavior:
- Consolidate fragmented regions where possible and improve bucket coverage.
- Intended to be cheap; safe to call between benchmark phases.
## Benchmarks (this run)
Environment: .NET 8 (Release), x64, single-thread, arena=3072 MiB. Results from StgSharpDebug/HlsfStressTest.
- Free throughput: 1KB x 262,144 -> 38,928,422.9 free/s
- Alloc+free pairs: 256B x 5,000,000 -> 25,006,976.9 ops/s
- Fragmentation cycle (~512MB): fast recovery in short cycle (see test for details)
Representative allocation throughput (higher is better):
| Size | Ops | alloc/s (no touch) | alloc/s (touch) |
|------|----:|--------------------:|----------------:|
| 64B | 2,000,000 | 8,780,467.7 | 17,524,920.4 |
| 1KB | 1,572,864 | 8,974,865.4 | 9,585,531.6 |
| 4KB | 393,216 | 7,340,392.2 | 6,849,319.4 |
| 32KB | 49,152 | 10,376,625.6 | 7,019,407.9 |
| 256KB | 6,144 | 16,041,775.5 | 12,842,809.4 |
| 512KB | 3,072 | 20,466,355.8 | 14,195,933.5 |
| 1MB | 1,536 | 16,916,299.6 | 13,568,904.6 |
| 2MB | 768 | 17,985,948.5 | 12,569,558.1 |
| 8MB | 192 | 13,426,573.4 | 11,034,482.8 |
| 32MB | 48 | 11,162,790.7 | 9,411,764.7 |
Notes:
- Peak no-touch at 2MB (~17.99M alloc/s). Peak touch near 64B (~17.52M alloc/s).
- Non-aligned sizes (e.g., 768B, 3KB) typically lower due to slicing/boundary handling.
## Detailed Design (summary)
-
- Size classes: levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] (64B x 4^level)
- Segments per level: 1x, 2x, 3x
- Buckets: index = 3 * level + segment (overflow bucket for level > 9)
- Metadata: Entry (external SLAB, position chain), BucketNode (user region, bucket chain)
- Position chain: circular doubly linked list anchored by spare sentinel
- Allocation: search buckets by level/segment with escalation; if none, carve 32MB from spare; set Alloc; slice remainder into buckets
- Slicing: align to level boundaries, pack contiguous blocks forward/backward to minimize overhead
- Free/Merge: bidirectional merge under boundary and level constraints; update buckets; optional level promotion
- Alignment: default 16B; internal blocks aligned by slicing/edge calculation
- Complexity: average O(1) with bounded levels and constant-time bucket ops
- `Entry` (external SLAB): linkage in position chain, `Position`, `Size`, `Level`, `State`
- Invariants: near neighbors must form a consistent double-linked ring; `Level` setter validates size vs. level window
- `BucketNode` (in user region): `EntryRef`, `NextLevel`, `PreviousLevel`, `IsInBucket`
- Position chain: circular doubly linked list anchored by sentinel `_spareMemory` (`State.Spare`)
### Allocation Flow
1) Compute `level` and `segment` from requested `size`
2) Try pop bucket (`TryPopLevelForAllocation(level, segment, out entry)`) and escalate:
- Advance `segment` 0¡ú1¡ú2, then `level++`, until `MaxLevel`
- If still empty: try pop from overflow bucket (L=10, seg=1) or `TryAllocateNew32MBBlock`
3) On success: bound-check the slice offset; set `entry` to `Alloc` using `SetEntryLevel`
4) If remainder exists in the source region, call `SliceMemory(entry->NextNear, remainder)` to produce free chunks into buckets
### Slicing Logic (`SliceMemory`)
- Inputs: `next` entry position, `sizeToSlice`
- Compute `offset` and `end` relative to base buffer; pick starting level by trailing-zero count of `offset`:
- `i = (BitOperations.TrailingZeroCount(offset) - 6) / 2`, clamped to `¡Ü MaxLevel`
- Three phases to cover [offset, end):
1) Forward levels i..N-1: for each level size `size` and `upper=size*4`, align to `edge=((offset+upper-1)/upper)*upper`, pack `count = ((min(edge,end) - offset) / size)` blocks
2) If `i > MaxLevel`: pack in largest upper window on tail
3) Backward i-1..0: fill remaining tail with smaller sizes
- For each packed region: allocate new `Entry`, insert before `next`, mark `Empty`, and push corresponding `BucketNode` into the `(level, segment)` bucket (`PushBucketToLevel(i, count-1, b)`)
### Free and Merge (`MergeAndRemoveFromBuckets`)
- Given an `Entry` to free (turned `Empty` by caller):
- Merge left while neighbor `p` is `Empty`, same `Level`, and within boundary (`AssertBoundary`)
- Remove `p` from bucket (`RemoveFromBucket`), expand current, fix `Position`, unlink `p`
- Merge right with neighbor `n` by the same rule
- Optionally promote `Level` if `Size ¡Ý levelSizeArray[originLevel] * 4`
- Ensure `BucketNode.EntryRef` points to the merged entry; caller re-enqueues into the proper bucket
### Buckets
- `PushBucketToLevel(level, segment, node)`: push to bucket head, set `IsInBucket`, maintain prev/next
- `TryPopLevel(level, segment, out node)`: pop head; if empty, return false
- `RemoveFromBucket(level, segment, node)`: unlink from middle or use `TryPopLevel` if it is the head
### Large Allocations and Spare Memory
- Requests > 32MB use `NativeMemory.AlignedAlloc(size, align)` and mark `State.Large`
- When buckets are empty, `TryAllocateNew32MBBlock` carves a 32MB region from `_spareMemory`, builds an `Empty` entry, and advances the spare pointer
### Alignment
- Default `_align = 16`; `AlignOfLevel(level)` returns `min(levelSizeArray[level], _align)`
- Internal blocks align to level boundaries by design of slicing/edge calculations
### Complexity and Invariants
- Allocate/free are O(1) on average with bounded levels (¡Ü10) and constant-time bucket ops
- Invariants guarded by exceptions (e.g., `HLSFPositionChainBreakException`) and pointer consistency checks
## Profiling (hotspots, current workloads)
- SliceMemory ~22% (self ~17%) ¡ª slicing and bucket enqueue dominate allocation path
- MergeAndRemoveFromBuckets ~13% (self ~12%) ¡ª merge + removal dominate free path
- Collect ~7.6% (self ~7.5%) ¡ª low overall cost
Focus: reduce per-slice math and bucket ops; add fast-paths for aligned regions; batch/lazy removals after merges.
StgSharp.Common/HighPerformance/Memory/SlabAllocationHandle.cs
+56
View File
@@ -0,0 +1,56 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe struct SlabAllocationHandle<T> : IDisposable where T: unmanaged
{
private readonly SlabAllocator<T> _allocator;
internal SlabAllocationHandle(SlabAllocator<T> allocator, bool locked)
{
_allocator = allocator;
Locked = locked;
}
public bool Locked { get; set; }
public void Dispose()
{
if (Locked)
{
_allocator.ExitBufferReading();
Locked = false;
}
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs
+126
View File
@@ -0,0 +1,126 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.Chunked"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private BufferStack<nuint> _buffers;
private BufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_buffers.Push(_currentBuffer);
_recycle = BufferStackBuilder.Create(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
// nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _currentCapacity)
{
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(newBuffer);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
public override void Dispose()
{
// NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer))
{
NativeMemory.Free((void*)buffer);
}
_buffers.Dispose();
_recycle.Dispose();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ChunkedSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs
+150
View File
@@ -0,0 +1,150 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.ChunkedConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private readonly BufferExpansionLock _lock = new();
private ConcurrentBufferStack<nuint> _buffers;
private ConcurrentBufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ConcurrentChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_recycle = BufferStackBuilder.CreateConcurrent(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
while (true)
{
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
nuint curIndex = Volatile.Read(ref _currentIndex);
_lock.EnterMetaDataRead();
try
{
if (curIndex >= _currentCapacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
nuint cap = Volatile.Read(ref _currentCapacity);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
_ = Interlocked.Exchange(ref _currentIndex, 0);
}
finally
{
_lock.ExitExpansionProcess();
}
}
curIndex = Volatile.Read(ref _currentIndex);
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer)) {
NativeMemory.Free((void*)buffer);
}
_lock.Dispose();
_buffers.Dispose();
_recycle.Dispose();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ConcurrentChunkedSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.SequencialConcurrent.cs
+158
View File
@@ -0,0 +1,158 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.SequencialConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private T* _buffer;
private readonly BufferExpansionLock _lock = new();
private readonly ConcurrentBufferStack<nuint> _stack;
private readonly int _elementSize;
private nuint _highestAllocated;
private ulong _capacity;
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
{
_elementSize = Unsafe.SizeOf<T>();
_capacity = initialCapacity;
_highestAllocated = 0;
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
int initCount = _capacity switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)_capacity / 4
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = (nuint)i;
}
_stack = BufferStackBuilder.CreateConcurrent(span);
_highestAllocated += (nuint)initCount;
}
internal T* BasePointer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _buffer;
}
public override nuint Allocate()
{
if (_stack.TryPop(out nuint index)) {
return index;
}
while (true)
{
index = Volatile.Read(ref _highestAllocated);
_lock.EnterMetaDataRead();
try
{
if (index >= _capacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
_capacity = Volatile.Read(ref _capacity) * 2;
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
_lock.EnterBufferCopy();
try
{
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
return index;
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
_stack.Dispose();
NativeMemory.Free(_buffer);
_lock.Dispose();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading()
{
_lock.EnterBufferRead();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading()
{
_lock.ExitBufferRead();
}
public override void Free(nuint index)
{
_stack.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
EnterBufferReading();
return new SlabAllocationHandle<T>(this, true);
}
~ConcurrentSequentialSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs
+121
View File
@@ -0,0 +1,121 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
namespace StgSharp.HighPerformance.Memory
{
public abstract class SlabAllocator<T> : IDisposable where T : unmanaged
{
public abstract nuint Allocate();
/// <summary>
/// Creates a new SLAB allocator instance with the specified configuration.
/// </summary>
/// <param name="count">
/// Initial number of objects that can be allocated.
/// </param>
/// <param name="layout">
/// Memory layout strategy for the SLAB allocator.
/// </param>
/// <param name="concurrentSupport">
/// Whether to enable concurrent access support.
/// </param>
/// <returns>
/// A new SLAB allocator instance.
/// </returns>
/// <exception cref="ArgumentOutOfRangeException">
/// Thrown when an invalid layout is specified.
/// </exception>
/// <exception cref="NotSupportedException">
/// Thrown when requesting unsupported configurations.
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para> While SLAB
/// /// allocators themselves may be designed for single-threaded use, allocated objects
/// are /// frequently shared across multiple threads in real-world applications. Due to
/// the /// special implementation of expandable concurrent sequential SLAB allocators,
/// they rely /// on internal expansion locks to ensure that memory buffer expansions
/// (which involve /// copying the entire buffer to a new location) do not occur while
/// other threads are /// reading from allocated regions.</para> <para><strong>
/// Sequential Layout Limitations: /// </strong><br /> Sequential layout SLAB allocators
/// cannot provide a single-threaded /// version because:<list
/// type="bullet"><item><description> They use contiguous memory /// layout that
/// requires buffer expansion when capacity is exceeded ///
/// </description></item><item><description> Buffer expansion involves copying all ///
/// existing data to a new memory location </description></item><item><description>///
/// Without proper synchronization, concurrent access to allocated objects during ///
/// expansion would cause memory corruption </description></item><item><description> The
/// /// expansion lock mechanism is essential even if allocation itself is single- ///
/// threaded</description></item></list></para> <para><strong> Chunked Layout: ///
/// </strong><br /> Chunked layout allocators can optionally provide single-threaded ///
/// versions since they don't require buffer expansion and copying. Each chunk is ///
/// allocated independently, avoiding the memory relocation issues present in sequential
/// /// layouts.</para>
/// </remarks>
public static SlabAllocator<T> Create(
nuint count,
SlabBufferLayout layout,
bool concurrentSupport = true)
{
if (concurrentSupport) {
return layout switch
{
SlabBufferLayout.Sequential => new ConcurrentSequentialSlabAllocator<T>(count),
SlabBufferLayout.Chunked => new ConcurrentChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
return layout switch
{
SlabBufferLayout.Sequential => throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."),
SlabBufferLayout.Chunked => new ChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
public abstract void Dispose();
public abstract void EnterBufferReading();
public abstract void ExitBufferReading();
public abstract void Free(nuint index);
public abstract SlabAllocationHandle<T> ReadAllocation();
}
public enum SlabBufferLayout
{
Sequential,
Chunked
}
}