youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize all existing files to LF

Browse Source
This commit is contained in:
Youmu Liu
2026-03-06 18:31:11 +08:00
parent 0a0af53af1
commit 2149bbeede
323 changed files with 113191 additions and 113191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/BufferStack.cs
+162 -162
View File
@@ -1,163 +1,163 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="BufferStack"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
public unsafe class BufferStack<T> : IDisposable where T: unmanaged
{
private byte* _buffer;
private ulong _count;
public BufferStack(int capacity)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
AllocateBuffer(capacity);
}
public int Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (int)_count;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
((Header*)_buffer)->Top = 0;
_count = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose()
{
Marshal.FreeHGlobal((nint)_buffer);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T Pop()
{
Header* header = (Header*)_buffer;
if (header->Top == 0) {
throw new InvalidOperationException("Stack underflow");
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
return dataStart[header->Top];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Push(T item)
{
Header* header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
Expand();
header = (Header*)_buffer;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[header->Top] = item;
header->Top++;
_count++;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPop(out T value)
{
Header* header = (Header*)_buffer;
if (header->Top == 0)
{
value = default;
return false;
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[header->Top];
return true;
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = (nuint)span.Length;
_count = (ulong)span.Length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = (nuint)capacity;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void Expand()
{
Header* header = (Header*)_buffer;
nuint newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
header = (Header*)_buffer;
header->Capacity = newCap;
}
private struct Header
{
public nuint Capacity;
public nuint Top;
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="BufferStack"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
public unsafe class BufferStack<T> : IDisposable where T: unmanaged
{
private byte* _buffer;
private ulong _count;
public BufferStack(int capacity)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
AllocateBuffer(capacity);
}
public int Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (int)_count;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
((Header*)_buffer)->Top = 0;
_count = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose()
{
Marshal.FreeHGlobal((nint)_buffer);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T Pop()
{
Header* header = (Header*)_buffer;
if (header->Top == 0) {
throw new InvalidOperationException("Stack underflow");
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
return dataStart[header->Top];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Push(T item)
{
Header* header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
Expand();
header = (Header*)_buffer;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[header->Top] = item;
header->Top++;
_count++;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPop(out T value)
{
Header* header = (Header*)_buffer;
if (header->Top == 0)
{
value = default;
return false;
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[header->Top];
return true;
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = (nuint)span.Length;
_count = (ulong)span.Length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = (nuint)capacity;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void Expand()
{
Header* header = (Header*)_buffer;
nuint newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
header = (Header*)_buffer;
header->Capacity = newCap;
}
private struct Header
{
public nuint Capacity;
public nuint Top;
}
}
}
StgSharp.Common/HighPerformance/Memory/ConcurrentBufferStack.cs
+308 -308
View File
@@ -1,308 +1,308 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="ConcurrentBufferStack"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
public static class BufferStackBuilder
{
// For BufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BufferStack<T> Create<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new BufferStack<T>(4); // Default small capacity
}
BufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
// For ConcurrentBufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ConcurrentBufferStack<T> CreateConcurrent<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new ConcurrentBufferStack<T>(4); // Default small capacity
}
ConcurrentBufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
}
[CollectionBuilder(builderType: typeof(BufferStackBuilder), methodName: nameof(BufferStackBuilder.CreateConcurrent))]
public unsafe class ConcurrentBufferStack<T> : IDisposable where T: unmanaged
{
private byte* _buffer;
private readonly BufferExpansionLock _lock = new();
private long _count;
public ConcurrentBufferStack()
{
AllocateBuffer(4);
}
public ConcurrentBufferStack(int capacity)
{
if (capacity <= 0) {
throw new ArgumentOutOfRangeException(nameof(capacity));
}
AllocateBuffer(capacity);
}
public ulong Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (ulong)Volatile.Read(ref _count);
}
public void Clear()
{
_lock.EnterExpansionProcess();
try
{
_lock.EnterBufferCopy(); // exclusive while mutating Top
try
{
Header* header = (Header*)_buffer;
header->Top = 0;
Volatile.Write(ref _count, 0);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
public void Dispose()
{
_lock.Dispose();
NativeMemory.Free(_buffer);
}
public T Pop()
{
while (true)
{
_lock.EnterBufferRead(); // prevent buffer being reallocated while reading/modifying Top
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0) {
throw new InvalidOperationException("Stack underflow");
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
_ = Interlocked.Decrement(ref _count);
return dataStart[oldTop - 1];
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
public void Push(T item)
{
while (true)
{
// Ensure capacity, expanding if needed
_lock.EnterMetaDataRead();
try
{
Header* header = (Header*)_buffer;
int topSnapshot = Volatile.Read(ref header->Top);
int cap = Volatile.Read(ref header->Capacity);
if (topSnapshot >= cap)
{
// Need expansion
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
int newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + ((nuint)newCap * (nuint)sizeof(T));
_lock.EnterBufferCopy();
try
{
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
((Header*)_buffer)->Capacity = newCap;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
finally
{
_lock.ExitExpansionProcess();
}
continue;
}
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
// Reserve and write under exclusive buffer lock to avoid consumers reading unwritten data
_lock.EnterBufferCopy();
try
{
Header* header = (Header*)_buffer;
int idx = header->Top; // reserve next slot
if (idx >= header->Capacity)
{
// lost race against expander or other producer; retry
continue;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[idx] = item; // write first
header->Top = idx + 1; // then publish by increasing top
_ = Interlocked.Increment(ref _count);
return;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
public bool TryPop(out T value)
{
while (true)
{
_lock.EnterBufferRead();
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0)
{
value = default;
return false;
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[oldTop - 1];
_ = Interlocked.Decrement(ref _count);
return true;
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, (nuint)header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = span.Length;
Volatile.Write(ref _count, span.Length);
}
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + (nuint)capacity * (nuint)sizeof(T);
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = capacity;
}
private struct Header
{
public int Capacity; // max capacity of the stack (number of elements)
public int Top; // next available index in the stack
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="ConcurrentBufferStack"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
public static class BufferStackBuilder
{
// For BufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BufferStack<T> Create<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new BufferStack<T>(4); // Default small capacity
}
BufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
// For ConcurrentBufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ConcurrentBufferStack<T> CreateConcurrent<T>(ReadOnlySpan<T> span) where T: unmanaged
{
if (span.IsEmpty)
{
return new ConcurrentBufferStack<T>(4); // Default small capacity
}
ConcurrentBufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
}
[CollectionBuilder(builderType: typeof(BufferStackBuilder), methodName: nameof(BufferStackBuilder.CreateConcurrent))]
public unsafe class ConcurrentBufferStack<T> : IDisposable where T: unmanaged
{
private byte* _buffer;
private readonly BufferExpansionLock _lock = new();
private long _count;
public ConcurrentBufferStack()
{
AllocateBuffer(4);
}
public ConcurrentBufferStack(int capacity)
{
if (capacity <= 0) {
throw new ArgumentOutOfRangeException(nameof(capacity));
}
AllocateBuffer(capacity);
}
public ulong Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (ulong)Volatile.Read(ref _count);
}
public void Clear()
{
_lock.EnterExpansionProcess();
try
{
_lock.EnterBufferCopy(); // exclusive while mutating Top
try
{
Header* header = (Header*)_buffer;
header->Top = 0;
Volatile.Write(ref _count, 0);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
public void Dispose()
{
_lock.Dispose();
NativeMemory.Free(_buffer);
}
public T Pop()
{
while (true)
{
_lock.EnterBufferRead(); // prevent buffer being reallocated while reading/modifying Top
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0) {
throw new InvalidOperationException("Stack underflow");
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
_ = Interlocked.Decrement(ref _count);
return dataStart[oldTop - 1];
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
public void Push(T item)
{
while (true)
{
// Ensure capacity, expanding if needed
_lock.EnterMetaDataRead();
try
{
Header* header = (Header*)_buffer;
int topSnapshot = Volatile.Read(ref header->Top);
int cap = Volatile.Read(ref header->Capacity);
if (topSnapshot >= cap)
{
// Need expansion
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
int newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + ((nuint)newCap * (nuint)sizeof(T));
_lock.EnterBufferCopy();
try
{
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
((Header*)_buffer)->Capacity = newCap;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
finally
{
_lock.ExitExpansionProcess();
}
continue;
}
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
// Reserve and write under exclusive buffer lock to avoid consumers reading unwritten data
_lock.EnterBufferCopy();
try
{
Header* header = (Header*)_buffer;
int idx = header->Top; // reserve next slot
if (idx >= header->Capacity)
{
// lost race against expander or other producer; retry
continue;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[idx] = item; // write first
header->Top = idx + 1; // then publish by increasing top
_ = Interlocked.Increment(ref _count);
return;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
public bool TryPop(out T value)
{
while (true)
{
_lock.EnterBufferRead();
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0)
{
value = default;
return false;
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[oldTop - 1];
_ = Interlocked.Decrement(ref _count);
return true;
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, (nuint)header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = span.Length;
Volatile.Write(ref _count, span.Length);
}
private void AllocateBuffer(int capacity)
{
nuint size = (nuint)sizeof(Header) + (nuint)capacity * (nuint)sizeof(T);
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = capacity;
}
private struct Header
{
public int Capacity; // max capacity of the stack (number of elements)
public int Top; // next available index in the stack
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs
+114 -114
View File
@@ -1,114 +1,114 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Drawing;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Allocator = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.Mathematics.Memory
{
public readonly unsafe struct HybridLayerSegregatedFitAllocationHandle
{
internal readonly Allocator.Entry* EntryHandle;
public readonly long AllocSize;
internal HybridLayerSegregatedFitAllocationHandle(Allocator.Entry* handle, long s)
{
EntryHandle = handle;
Pointer = (byte*)handle->Position;
AllocSize = s;
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
}
public readonly ref byte this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, AllocSize);
return ref Pointer[index];
}
}
public readonly byte* Pointer { get; init; }
public readonly Span<byte> BufferHandle
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => new(Pointer, (int)AllocSize);
}
public readonly Enumerator GetEnumerator()
{
return new Enumerator(this);
}
public ref struct Enumerator : IEnumerator<byte>
{
private HybridLayerSegregatedFitAllocationHandle _handle;
private uint _index = ~0u;
internal Enumerator(HybridLayerSegregatedFitAllocationHandle span)
{
_handle = span;
Span = span.BufferHandle;
}
public ref byte RefCurrent => ref _handle[_index];
public Span<byte> Span { get; init; }
public readonly void Dispose() { }
public bool MoveNext()
{
_index++;
return _index < _handle.AllocSize;
}
public void Reset()
{
_index = 0;
}
object IEnumerator.Current => RefCurrent;
byte IEnumerator<byte>.Current => RefCurrent;
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Drawing;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Allocator = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.Mathematics.Memory
{
public readonly unsafe struct HybridLayerSegregatedFitAllocationHandle
{
internal readonly Allocator.Entry* EntryHandle;
public readonly long AllocSize;
internal HybridLayerSegregatedFitAllocationHandle(Allocator.Entry* handle, long s)
{
EntryHandle = handle;
Pointer = (byte*)handle->Position;
AllocSize = s;
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
}
public readonly ref byte this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, AllocSize);
return ref Pointer[index];
}
}
public readonly byte* Pointer { get; init; }
public readonly Span<byte> BufferHandle
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => new(Pointer, (int)AllocSize);
}
public readonly Enumerator GetEnumerator()
{
return new Enumerator(this);
}
public ref struct Enumerator : IEnumerator<byte>
{
private HybridLayerSegregatedFitAllocationHandle _handle;
private uint _index = ~0u;
internal Enumerator(HybridLayerSegregatedFitAllocationHandle span)
{
_handle = span;
Span = span.BufferHandle;
}
public ref byte RefCurrent => ref _handle[_index];
public Span<byte> Span { get; init; }
public readonly void Dispose() { }
public bool MoveNext()
{
_index++;
return _index < _handle.AllocSize;
}
public void Reset()
{
_index = 0;
}
object IEnumerator.Current => RefCurrent;
byte IEnumerator<byte>.Current => RefCurrent;
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+234 -234
View File
@@ -1,234 +1,234 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Numerics;
using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
public hlsfHandle Alloc(long size)
{
if (size > _maxSize) {
throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
}
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
int i = level, s = segmentIndex;
Entry* handle;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out handle))
{
break;
} else if (TryAllocateNew32MBBlock(out handle))
{
break;
}
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
}
if (handle != null)
{
int requestedOffset = (segmentIndex + 1) * _levelSizeArray[level];
if (requestedOffset > handle->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
long remainder = handle->Size - requestedOffset;
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
handle->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(handle->NextNear, remainder);
}
}
return new hlsfHandle(handle, size);
}
/// <summary>
/// Slice the memory block into smaller blocks and add them to the appropriate buckets
/// Notice: this method is not responsible for freeing the original BucketNode
/// </summary>
/// <param name="e">
///
/// </param>
/// <param name="sizeToSlice">
///
/// </param>
private void SliceMemory(Entry* e, long sizeToSlice)
{
Entry* next = e;
long remain = sizeToSlice; // remained size to slice
long offset = (long)e->Position - sizeToSlice - (long)m_Buffer; // offset of buffer to slice
long end = (long)e->Position - (long)m_Buffer; // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
i = int.Min(10, i);
for (; i < _levelSizeArray.Length; i++)
{
int size = _levelSizeArray[i];
int upper = size * 4;
long edge = (offset + upper - 1) / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
if (i > MaxLevel)
{
int size = _levelSizeArray[^1];
int upper = size * 4;
long edge = end / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / upper);
if (count > 0)
{
long newSize = (long)count * (long)upper;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
if (remain <= 0) {
return;
}
}
}
for (i--; i >= 0; i--)
{
int size = _levelSizeArray[i];
long edge = end / size * size;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
b->NextLevel = null;
b->PreviousLevel = null;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
}
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew32MBBlock(out Entry* handle)
{
nuint pos = _spareMemory->Position;
long remain = _spareMemory->Size;
if (remain <= 0)
{
handle = null;
return false;
}
long size = remain > 32 * 1024 * 1024L ? 32 * 1024 * 1024L : remain;
// build new entry
Entry* e = (Entry*)_entries.Allocate();
SetEntryLevel(e, pos, size);
InsertEntryBefore(e, _spareMemory);
e->State = EntryState.Empty;
// set fields of spare memory
_spareMemory->Size -= size;
_spareMemory->Position += (nuint)size;
handle = e;
return true;
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Numerics;
using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
public hlsfHandle Alloc(long size)
{
if (size > _maxSize) {
throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
}
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
int i = level, s = segmentIndex;
Entry* handle;
while (!TryPopLevelForAllocation(i, s, out handle))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out handle))
{
break;
} else if (TryAllocateNew32MBBlock(out handle))
{
break;
}
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
}
}
if (handle != null)
{
int requestedOffset = (segmentIndex + 1) * _levelSizeArray[level];
if (requestedOffset > handle->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
long remainder = handle->Size - requestedOffset;
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
handle->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(handle->NextNear, remainder);
}
}
return new hlsfHandle(handle, size);
}
/// <summary>
/// Slice the memory block into smaller blocks and add them to the appropriate buckets
/// Notice: this method is not responsible for freeing the original BucketNode
/// </summary>
/// <param name="e">
///
/// </param>
/// <param name="sizeToSlice">
///
/// </param>
private void SliceMemory(Entry* e, long sizeToSlice)
{
Entry* next = e;
long remain = sizeToSlice; // remained size to slice
long offset = (long)e->Position - sizeToSlice - (long)m_Buffer; // offset of buffer to slice
long end = (long)e->Position - (long)m_Buffer; // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
i = int.Min(10, i);
for (; i < _levelSizeArray.Length; i++)
{
int size = _levelSizeArray[i];
int upper = size * 4;
long edge = (offset + upper - 1) / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
if (i > MaxLevel)
{
int size = _levelSizeArray[^1];
int upper = size * 4;
long edge = end / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / upper);
if (count > 0)
{
long newSize = (long)count * (long)upper;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
if (remain <= 0) {
return;
}
}
}
for (i--; i >= 0; i--)
{
int size = _levelSizeArray[i];
long edge = end / size * size;
int count = (int)((long.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
b->EntryRef = newEntry;
b->NextLevel = null;
b->PreviousLevel = null;
PushBucketToLevel(i, count - 1, b);
offset += newSize;
remain -= newSize;
}
if (remain <= 0) {
return;
}
}
}
/// <summary>
/// Try to allocate a new 16MB block when all buckets are empty
/// </summary>
/// <param name="handle">
/// Output handle to the allocated Entry
/// </param>
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew32MBBlock(out Entry* handle)
{
nuint pos = _spareMemory->Position;
long remain = _spareMemory->Size;
if (remain <= 0)
{
handle = null;
return false;
}
long size = remain > 32 * 1024 * 1024L ? 32 * 1024 * 1024L : remain;
// build new entry
Entry* e = (Entry*)_entries.Allocate();
SetEntryLevel(e, pos, size);
InsertEntryBefore(e, _spareMemory);
e->State = EntryState.Empty;
// set fields of spare memory
_spareMemory->Size -= size;
_spareMemory->Position += (nuint)size;
handle = e;
return true;
}
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
entry = bucketNode->EntryRef;
return entry != null;
}
entry = null;
return false;
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+171 -171
View File
@@ -1,171 +1,171 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Reflection.Metadata;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private void PushBucketToLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
node->IsInBucket = true;
node->NextLevel = null;
node->PreviousLevel = null;
// Console.WriteLine($"Push to empty level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}");
return;
} else
{
BucketNode* head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = head;
node->IsInBucket = true;
_bucketHeads[index] = node;
// Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}");
if (headNode != null)
{
headNode->PreviousLevel = node;
}
return;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = (BucketNode*)null;
node->PreviousLevel = (BucketNode*)null;
node->IsInBucket = false;
return;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex;
BucketNode* head = _bucketHeads[index];
if (head == null)
{
node = null;
return false;
}
// Console.WriteLine($"level popped from level {levelIndex} {sizeIndex}, node is at {(ulong)head}");
BucketNode* next = head->NextLevel;
if (next == null)
{
_bucketHeads[index] = null;
node = head;
return true;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = null;
if (next != null) {
next->PreviousLevel = null;
}
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
return ok;
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = null;
node->PreviousLevel = null;
node->IsInBucket = false;
node->EntryRef = null;
return true;
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit)]
internal struct BucketNode
{
[FieldOffset(0)] internal Entry* EntryRef; // Pointer to corresponding Entry
// [FieldOffset(16)] internal ulong LockBuffer;
[FieldOffset(8)] internal BucketNode* NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(24)] internal BucketNode* PreviousLevel; // Bucket stack linked list: previous pointer
[FieldOffset(32)] internal bool IsInBucket; // Whether in bucket
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Reflection.Metadata;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private void PushBucketToLevel(int levelIndex, int sizeIndex, BucketNode* node)
{
if (node == null || node->EntryRef == null) {
return;
}
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
node->IsInBucket = true;
node->NextLevel = null;
node->PreviousLevel = null;
// Console.WriteLine($"Push to empty level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}");
return;
} else
{
BucketNode* head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = head;
node->IsInBucket = true;
_bucketHeads[index] = node;
// Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}");
if (headNode != null)
{
headNode->PreviousLevel = node;
}
return;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
return;
}
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = (BucketNode*)null;
node->PreviousLevel = (BucketNode*)null;
node->IsInBucket = false;
return;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
{
Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex;
BucketNode* head = _bucketHeads[index];
if (head == null)
{
node = null;
return false;
}
// Console.WriteLine($"level popped from level {levelIndex} {sizeIndex}, node is at {(ulong)head}");
BucketNode* next = head->NextLevel;
if (next == null)
{
_bucketHeads[index] = null;
node = head;
return true;
}
_bucketHeads[index] = head->NextLevel;
node = head;
node->IsInBucket = false;
node->NextLevel = null;
if (next != null) {
next->PreviousLevel = null;
}
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
{
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
if (_bucketHeads[index] == node)
{
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
return ok;
}
BucketNode* previous = node->PreviousLevel;
BucketNode* next = node->NextLevel;
if (previous != null) {
previous->NextLevel = next;
}
if (next != null) {
next->PreviousLevel = previous;
}
node->NextLevel = null;
node->PreviousLevel = null;
node->IsInBucket = false;
node->EntryRef = null;
return true;
}
/// <summary>
/// BucketNode structure - specialized node for bucket linked lists Stored in main
/// allocation area (user-overwritable region)
/// </summary>
[StructLayout(LayoutKind.Explicit)]
internal struct BucketNode
{
[FieldOffset(0)] internal Entry* EntryRef; // Pointer to corresponding Entry
// [FieldOffset(16)] internal ulong LockBuffer;
[FieldOffset(8)] internal BucketNode* NextLevel; // Bucket stack linked list: next pointer
[FieldOffset(24)] internal BucketNode* PreviousLevel; // Bucket stack linked list: previous pointer
[FieldOffset(32)] internal bool IsInBucket; // Whether in bucket
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs
+121 -121
View File
@@ -1,122 +1,122 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Debug"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
public sealed class HLSFPositionChainBreakException : Exception
{
public HLSFPositionChainBreakException() : base("Position chain of an HLSF allocator is broken") { }
}
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private object _lock = new object();
public void DisplayEntryLayout(bool singleStep)
{
lock (_lock)
{
Entry* cur = _spareMemory;
while (cur != null)
{
Console.WriteLine(EntryToString(cur));
cur = cur->PreviousNear;
if (singleStep) {
Console.ReadKey();
}
}
}
}
public void DumpPositionChain(bool singleStep = false, int maxCount = 4096)
{
lock (_lock)
{
try
{
Entry* cur = _spareMemory;
int cnt = 0;
if (cur != null) {
cur = cur->NextNear;
}
Console.WriteLine(EntryToString(_spareMemory));
while (cur != null && cur != _spareMemory && cnt < maxCount)
{
Console.WriteLine(EntryToString(cur));
cnt++;
if (singleStep) {
Console.ReadKey();
}
cur = cur->NextNear;
}
if (cur == _spareMemory)
{
Console.WriteLine("(reached sentinel)");
} else if (cnt >= maxCount) {
Console.WriteLine("(stopped at maxCount)");
}
}
catch (Exception ex)
{
Console.WriteLine($"[DumpPositionChain-forward] Exception: {ex.Message}");
}
}
}
public IEnumerable<string> ShowEntryLayout()
{
lock (_lock)
{
Stack<string> layout = [];
Entry* cur = _spareMemory;
while (cur != null)
{
layout.Push(EntryToString(cur));
cur = cur->PreviousNear;
}
return layout;
}
}
private static string EntryToString(Entry* e)
{
BucketNode* b = (BucketNode*)e->Position;
return $"{(ulong)e->PreviousNear}<-prev- Position:{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -next->{(ulong)e->NextNear}";
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Debug"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
public sealed class HLSFPositionChainBreakException : Exception
{
public HLSFPositionChainBreakException() : base("Position chain of an HLSF allocator is broken") { }
}
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private object _lock = new object();
public void DisplayEntryLayout(bool singleStep)
{
lock (_lock)
{
Entry* cur = _spareMemory;
while (cur != null)
{
Console.WriteLine(EntryToString(cur));
cur = cur->PreviousNear;
if (singleStep) {
Console.ReadKey();
}
}
}
}
public void DumpPositionChain(bool singleStep = false, int maxCount = 4096)
{
lock (_lock)
{
try
{
Entry* cur = _spareMemory;
int cnt = 0;
if (cur != null) {
cur = cur->NextNear;
}
Console.WriteLine(EntryToString(_spareMemory));
while (cur != null && cur != _spareMemory && cnt < maxCount)
{
Console.WriteLine(EntryToString(cur));
cnt++;
if (singleStep) {
Console.ReadKey();
}
cur = cur->NextNear;
}
if (cur == _spareMemory)
{
Console.WriteLine("(reached sentinel)");
} else if (cnt >= maxCount) {
Console.WriteLine("(stopped at maxCount)");
}
}
catch (Exception ex)
{
Console.WriteLine($"[DumpPositionChain-forward] Exception: {ex.Message}");
}
}
}
public IEnumerable<string> ShowEntryLayout()
{
lock (_lock)
{
Stack<string> layout = [];
Entry* cur = _spareMemory;
while (cur != null)
{
layout.Push(EntryToString(cur));
cur = cur->PreviousNear;
}
return layout;
}
}
private static string EntryToString(Entry* e)
{
BucketNode* b = (BucketNode*)e->Position;
return $"{(ulong)e->PreviousNear}<-prev- Position:{(ulong)e} Bucket:{{{(ulong)e->Position},{e->Level},{e->Size}}} -next->{(ulong)e->NextNear}";
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs
+129 -129
View File
@@ -1,129 +1,129 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.Mathematics.Memory
{
public enum FreePolicy
{
FullCollect,
LazyCollect,
NoCollect
}
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Minimal collect: linear scan, merge contiguous empty Blocks into one, push merged head
/// back to bucket. Caller-responsible design: no slicing or redistribution.
/// </summary>
public void Collect()
{
Entry* cur = _spareMemory->NextNear;
long size = 0;
while (cur != null && cur != _spareMemory)
{
// cur is not empty, skip
if (cur->State != EntryState.Empty)
{
cur = cur->NextNear;
continue;
}
// remove cur from bucket
do
{
size += cur->Size;
Entry* next = cur->NextNear;
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while (cur != null && cur != _spareMemory && cur->State == EntryState.Empty);
if (size > 0) {
SliceMemory(cur, size);
}
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(hlsfHandle handle, FreePolicy collectPolicy = FreePolicy.NoCollect)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
if (e->State != EntryState.Alloc) {
return;
}
// Mark as empty so it can be merged
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
// Calculate final level and segment for the merged block
long finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
// Console.WriteLine("one block freed");
if (collectPolicy == FreePolicy.FullCollect)
{
Collect();
}
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Reflection.Emit;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocationHandle;
namespace StgSharp.Mathematics.Memory
{
public enum FreePolicy
{
FullCollect,
LazyCollect,
NoCollect
}
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Minimal collect: linear scan, merge contiguous empty Blocks into one, push merged head
/// back to bucket. Caller-responsible design: no slicing or redistribution.
/// </summary>
public void Collect()
{
Entry* cur = _spareMemory->NextNear;
long size = 0;
while (cur != null && cur != _spareMemory)
{
// cur is not empty, skip
if (cur->State != EntryState.Empty)
{
cur = cur->NextNear;
continue;
}
// remove cur from bucket
do
{
size += cur->Size;
Entry* next = cur->NextNear;
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while (cur != null && cur != _spareMemory && cur->State == EntryState.Empty);
if (size > 0) {
SliceMemory(cur, size);
}
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(hlsfHandle handle, FreePolicy collectPolicy = FreePolicy.NoCollect)
{
Entry* e = handle.EntryHandle;
if (e == null) {
return;
}
if (e->State != EntryState.Alloc) {
return;
}
// Mark as empty so it can be merged
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
// Calculate final level and segment for the merged block
long finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
// Console.WriteLine("one block freed");
if (collectPolicy == FreePolicy.FullCollect)
{
Collect();
}
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs
+92 -92
View File
@@ -1,92 +1,92 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Level"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private const int MaxLevel = 9;
private int[] _levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216];
public int AlignOfLevel(int bucketLevel)
{
int blockSize = _levelSizeArray[bucketLevel];
return int.Min(blockSize, _align);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(long size)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
* Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level)
*
* For size S, we need: 2^(6+2*level) >= S
* So: 6+2*level >= log2(S)
* Therefore: level >= (log2(S) - 6) / 2
*
* We use hardware LZCNT instruction via BitOperations.LeadingZeroCount
* log2(S) = 63 - LeadingZeroCount(S) for 64-bit values
*/
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return level;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(uint size)
{
return GetLevelFromSize(size);
}
}
public enum CommonMemoryAlign : int
{
XMMLevel = 16,
YMMLevel = 32,
ZMMLevel = 64,
ArmNEONLevel = 16,
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Level"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private const int MaxLevel = 9;
private int[] _levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216];
public int AlignOfLevel(int bucketLevel)
{
int blockSize = _levelSizeArray[bucketLevel];
return int.Min(blockSize, _align);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(long size)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
* Pattern: 64 * 4^level = 2^6 * 2^(2*level) = 2^(6+2*level)
*
* For size S, we need: 2^(6+2*level) >= S
* So: 6+2*level >= log2(S)
* Therefore: level >= (log2(S) - 6) / 2
*
* We use hardware LZCNT instruction via BitOperations.LeadingZeroCount
* log2(S) = 63 - LeadingZeroCount(S) for 64-bit values
*/
int log2Size = 63 - BitOperations.LeadingZeroCount((ulong)size);
// Calculate level: level = max(0, (log2Size - 6) / 2)
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return level;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(uint size)
{
return GetLevelFromSize(size);
}
}
public enum CommonMemoryAlign : int
{
XMMLevel = 16,
YMMLevel = 32,
ZMMLevel = 64,
ArmNEONLevel = 16,
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+208 -208
View File
@@ -1,208 +1,208 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(Entry* current, Entry* other)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return other->Position + (nuint)other->Size == current->Position ||
current->Position + (nuint)current->Size == other->Position;
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
private bool AssertBoundary(Entry* previous, Entry* current, int level)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = _levelSizeArray[level] * 4;
ulong prevHead = previous->Position - (ulong)m_Buffer,
currHead = current->Position - (ulong)m_Buffer;
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int DetermineSegmentIndex(int[] levelSizeArray, long size, int level)
{
level = int.Min(level, MaxLevel);
return (int)((size - 1) / levelSizeArray[level]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryAfter(Entry* entry, Entry* position)
{
Entry* next = position->NextNear;
entry->NextNear = next;
entry->PreviousNear = position;
next->PreviousNear = entry;
position->NextNear = entry;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryBefore(Entry* entry, Entry* position)
{
Entry* prev = position->PreviousNear;
entry->NextNear = position;
entry->PreviousNear = prev;
prev->NextNear = entry;
position->PreviousNear = entry;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
long size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
do
{
// previous direction merge
Entry* p = entry->PreviousNear;
if (p == null ||
p == _spareMemory ||
p->State != EntryState.Empty ||
p->Level != entry->Level ||
!AssertBoundary(p, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
entry->Size += p->Size;
entry->Position = p->Position;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
do
{
// next direction merge
Entry* n = entry->NextNear;
if (n == null ||
n == _spareMemory ||
n->State != EntryState.Empty ||
n->Level != entry->Level ||
!AssertBoundary(n, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
entry->Size += n->Size;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += entry->Size >= _levelSizeArray[originLevel] * 4 ? 1 : 0;
entry->Level &= 15;
BucketNode* bucket = (BucketNode*)entry->Position;
bucket->EntryRef = entry;
}
/// <summary>
/// Remove node from position linked list, this method does not recycle a BucketNode from
/// size bucket
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
Entry* prev = entry->PreviousNear;
Entry* next = entry->NextNear;
prev->NextNear = next;
next->PreviousNear = prev;
// Clear node pointers
entry->NextNear = null;
entry->PreviousNear = null;
// Console.WriteLine("entry recycled.");
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryLevel(Entry* entry, nuint position, long size)
{
entry ->Position = position;
entry->Size = size;
entry->Level = GetLevelFromSize(size);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal long Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(28)] public nuint Position; // Memory block position
}
internal enum EntryState : uint
{
Spare,
Empty,
Alloc,
Large
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/// <summary>
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(Entry* current, Entry* other)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return other->Position + (nuint)other->Size == current->Position ||
current->Position + (nuint)current->Size == other->Position;
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
private bool AssertBoundary(Entry* previous, Entry* current, int level)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = _levelSizeArray[level] * 4;
ulong prevHead = previous->Position - (ulong)m_Buffer,
currHead = current->Position - (ulong)m_Buffer;
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int DetermineSegmentIndex(int[] levelSizeArray, long size, int level)
{
level = int.Min(level, MaxLevel);
return (int)((size - 1) / levelSizeArray[level]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryAfter(Entry* entry, Entry* position)
{
Entry* next = position->NextNear;
entry->NextNear = next;
entry->PreviousNear = position;
next->PreviousNear = entry;
position->NextNear = entry;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryBefore(Entry* entry, Entry* position)
{
Entry* prev = position->PreviousNear;
entry->NextNear = position;
entry->PreviousNear = prev;
prev->NextNear = entry;
position->PreviousNear = entry;
}
/// <summary>
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
return;
}
long size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
do
{
// previous direction merge
Entry* p = entry->PreviousNear;
if (p == null ||
p == _spareMemory ||
p->State != EntryState.Empty ||
p->Level != entry->Level ||
!AssertBoundary(p, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)p->Size} merged to {(ulong)entry->Size} previous");
entry->Size += p->Size;
entry->Position = p->Position;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
do
{
// next direction merge
Entry* n = entry->NextNear;
if (n == null ||
n == _spareMemory ||
n->State != EntryState.Empty ||
n->Level != entry->Level ||
!AssertBoundary(n, entry, originLevel))
{
break;
}
// Console.WriteLine($"{(ulong)n->Size} merged to {(ulong)entry->Size} next");
entry->Size += n->Size;
RemoveFromBucket(originLevel, DetermineSegmentIndex(_levelSizeArray, n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += entry->Size >= _levelSizeArray[originLevel] * 4 ? 1 : 0;
entry->Level &= 15;
BucketNode* bucket = (BucketNode*)entry->Position;
bucket->EntryRef = entry;
}
/// <summary>
/// Remove node from position linked list, this method does not recycle a BucketNode from
/// size bucket
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
{
Entry* prev = entry->PreviousNear;
Entry* next = entry->NextNear;
prev->NextNear = next;
next->PreviousNear = prev;
// Clear node pointers
entry->NextNear = null;
entry->PreviousNear = null;
// Console.WriteLine("entry recycled.");
_entries.Free((nuint)entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryLevel(Entry* entry, nuint position, long size)
{
entry ->Position = position;
entry->Size = size;
entry->Level = GetLevelFromSize(size);
}
/// <summary>
/// Entry structure - streamlined version (only position and state information) Stored in
/// external SLAB
/// </summary>
[StructLayout(LayoutKind.Explicit, Size = 40)]
internal struct Entry
{
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal long Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(28)] public nuint Position; // Memory block position
}
internal enum EntryState : uint
{
Spare,
Empty,
Alloc,
Large
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
+178 -178
View File
@@ -1,178 +1,178 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
// Bucket system field - directly in main allocator class
private readonly BucketNode *[] _bucketHeads;
// _spareMemory is also the beginning position of allocator
private Entry *_spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly long _maxSize;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _entries;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
_size = byteSize;
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
_bucketHeads = new BucketNode *[_levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Spare;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
_maxSize = 32U * 1024U * 1024U;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)
{
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
_size = byteSize;
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode *[(levelCount * 3) + 1];
int size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
_levelSizeArray[i] = size;
}
_maxSize = maxBlockSize;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
_bucketHeads = new BucketNode *[_levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
_maxSize = 32U * 1024U * 1024U;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align, long maxBlockSize)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode *[levelCount * 3 + 1];
int size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
_levelSizeArray[i] = size;
}
_maxSize = maxBlockSize;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_entries.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
namespace StgSharp.Mathematics.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
// Bucket system field - directly in main allocator class
private readonly BucketNode *[] _bucketHeads;
// _spareMemory is also the beginning position of allocator
private Entry *_spareMemory;
private readonly byte* m_Buffer;
private bool disposedValue;
private readonly int _align;
private readonly long _maxSize;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _entries;
public HybridLayerSegregatedFitAllocator(nuint byteSize)
{
if (byteSize < 64) {
throw new InvalidOperationException();
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
_size = byteSize;
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
_bucketHeads = new BucketNode *[_levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Spare;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
_maxSize = 32U * 1024U * 1024U;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)
{
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, 16);
_align = 16;
_size = byteSize;
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode *[(levelCount * 3) + 1];
int size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
_levelSizeArray[i] = size;
}
_maxSize = maxBlockSize;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte *)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
_bucketHeads = new BucketNode *[_levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
_maxSize = 32U * 1024U * 1024U;
}
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align, long maxBlockSize)
{
if (align < 16 || (align & (align - 1)) != 0) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = SlabAllocator<Entry>.Create(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte*)NativeMemory.AlignedAlloc(byteSize, (nuint)align);
_align = align;
_size = byteSize;
_spareMemory = (Entry *)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode *[levelCount * 3 + 1];
int size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
_levelSizeArray[i] = size;
}
_maxSize = maxBlockSize;
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing) {
_entries.Dispose();
}
NativeMemory.AlignedFree(m_Buffer);
disposedValue = true;
}
}
~HybridLayerSegregatedFitAllocator()
{
Dispose(disposing:false);
}
}
}
StgSharp.Common/HighPerformance/Memory/InroductionToHLSF.md
+136 -136
View File
@@ -1,137 +1,137 @@
# HLSF (Hybrid Layer Segregated Fit) Allocator
A constant-time (O(1)) allocator for real-time, high-throughput scenarios. It uses layered size classes with segmented buckets and stores all metadata externally for clean user memory and better cache locality.
## Key Points
- O(1) allocate/free with 10 levels: 64B x 16MB; each level split into 1x/2x/3x segments
- External metadata (`Entry`, `BucketNode`) in dedicated SLABs; user blocks have no headers
- Configurable alignment (>=16B); maximum block size is configurable at construction time via the `maxBlockSize` parameter (default 32MB, 10 levels). The level array and bucket table are dynamically extended when a larger `maxBlockSize` is specified. There is no implicit fallback to `NativeMemory` for oversized requests.
- Single-threaded implementation (external sync required for multi-threaded use)
## Quick Use
```csharp
using var hlsf = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024);
var h = hlsf.Alloc(1024);
Span<byte> s = h.BufferHandle; s[0] = 42;
hlsf.Free(h);
```
## Core APIs
### Alloc(uint size)
- Purpose: Allocate a block of at least `size` bytes.
- Input: `size` in bytes. Must not exceed the `maxBlockSize` configured at construction time (default 32MB). If `size > _maxSize`, an `InvalidOperationException` is thrown.
- Output: `HybridLayerSegregatedFitAllocationHandle` with pointer access and `Span<byte>`.
- Behavior:
- Classify into level/segment and pop a matching bucket; escalate segments and levels if empty.
- Fallback to overflow bucket or carve 32MB from spare memory if all buckets are empty.
- Mark selected `Entry` as Alloc; slice any remainder into `Empty` entries and push to buckets.
- There is no implicit fallback to `NativeMemory.AlignedAlloc` for oversized requests — all allocations are served from the pre-allocated arena. The level array and bucket table are dynamically sized based on the `maxBlockSize` parameter.
### Free(HybridLayerSegregatedFitAllocationHandle handle)
- Purpose: Free a previously allocated block.
- Behavior:
- Mark the corresponding `Entry` as `Empty`.
- Merge with adjacent `Empty` entries in both directions (boundary and level constraints).
- Remove merged neighbors from buckets, update position chain, and optionally promote level.
- Re-enqueue the final `Entry` into the appropriate bucket.
### Collect()
- Purpose: Reclaim and rebalance free entries without touching allocated blocks.
- Behavior:
- Consolidate fragmented regions where possible and improve bucket coverage.
- Intended to be cheap; safe to call between benchmark phases.
### BenchmarkDotNet (HLSF vs Libc)
Environment: .NET 8.0.23 (Release), x64 RyuJIT AVX2, Windows 11, AMD Ryzen 7 6800H.
| Method | Mean | StdDev | Ratio | Allocated |
|------- |-----:|------:|------:|----------:|
| Hlsf | 838.4 µs | 40.6 µs | 1.00 | 64 B |
| Libc | 849.8 µs | 100.2 µs | 1.01 | 64 B |
Interpretation: HLSF matches libc allocation throughput under the same workload, with slightly tighter dispersion in this run.
> Note: BenchmarkDotNet also warned that the minimum iteration time was under 1ms; for more stable statistics, increase per-iteration work (e.g., higher `AllocationCount`).
## Detailed Design (summary)
-
- Size classes: The default level size array is [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] (10 levels, 64B × 4^level). When `maxBlockSize` is specified at construction and exceeds the default range, additional levels are appended (each 4× the previous) to cover the requested maximum block size.
- Segments per level: 1x, 2x, 3x
- Buckets: index = 3 * level + segment (overflow bucket for level > 9)
- Metadata: Entry (external SLAB, position chain), BucketNode (user region, bucket chain)
- Position chain: circular doubly linked list anchored by spare sentinel
- Allocation: search buckets by level/segment with escalation; if none, carve 32MB from spare; set Alloc; slice remainder into buckets
- Slicing: align to level boundaries, pack contiguous blocks forward/backward to minimize overhead
- Free/Merge: bidirectional merge under boundary and level constraints; update buckets; optional level promotion
- Alignment: default 16B; internal blocks aligned by slicing/edge calculation
- Complexity: average O(1) with bounded levels and constant-time bucket ops
- `Entry` (external SLAB): linkage in position chain, `Position`, `Size`, `Level`, `State`
- Invariants: near neighbors must form a consistent double-linked ring; `Level` setter validates size vs. level window
- `BucketNode` (in user region): `EntryRef`, `NextLevel`, `PreviousLevel`, `IsInBucket`
- Position chain: circular doubly linked list anchored by sentinel `_spareMemory` (`State.Spare`)
### Allocation Flow
1) Compute `level` and `segment` from requested `size`
2) Try pop bucket (`TryPopLevelForAllocation(level, segment, out entry)`) and escalate:
- Advance `segment` 0��1��2, then `level++`, until `MaxLevel`
- If still empty: try pop from overflow bucket (L=10, seg=1) or `TryAllocateNew32MBBlock`
3) On success: bound-check the slice offset; set `entry` to `Alloc` using `SetEntryLevel`
4) If remainder exists in the source region, call `SliceMemory(entry->NextNear, remainder)` to produce free chunks into buckets
### Slicing Logic (`SliceMemory`)
- Inputs: `next` entry position, `sizeToSlice`
- Compute `offset` and `end` relative to base buffer; pick starting level by trailing-zero count of `offset`:
- `i = (BitOperations.TrailingZeroCount(offset) - 6) / 2`, clamped to `�� MaxLevel`
- Three phases to cover [offset, end):
1) Forward levels i..N-1: for each level size `size` and `upper=size*4`, align to `edge=((offset+upper-1)/upper)*upper`, pack `count = ((min(edge,end) - offset) / size)` blocks
2) If `i > MaxLevel`: pack in largest upper window on tail
3) Backward i-1..0: fill remaining tail with smaller sizes
- For each packed region: allocate new `Entry`, insert before `next`, mark `Empty`, and push corresponding `BucketNode` into the `(level, segment)` bucket (`PushBucketToLevel(i, count-1, b)`)
### Free and Merge (`MergeAndRemoveFromBuckets`)
- Given an `Entry` to free (turned `Empty` by caller):
- Merge left while neighbor `p` is `Empty`, same `Level`, and within boundary (`AssertBoundary`)
- Remove `p` from bucket (`RemoveFromBucket`), expand current, fix `Position`, unlink `p`
- Merge right with neighbor `n` by the same rule
- Optionally promote `Level` if `Size �� levelSizeArray[originLevel] * 4`
- Ensure `BucketNode.EntryRef` points to the merged entry; caller re-enqueues into the proper bucket
### Buckets
- `PushBucketToLevel(level, segment, node)`: push to bucket head, set `IsInBucket`, maintain prev/next
- `TryPopLevel(level, segment, out node)`: pop head; if empty, return false
- `RemoveFromBucket(level, segment, node)`: unlink from middle or use `TryPopLevel` if it is the head
### Large Allocations and Spare Memory
- Requests exceeding the configured `maxBlockSize` are rejected with `InvalidOperationException`. All allocations are served from the pre-allocated arena.
- When all buckets are empty, `TryAllocateNew32MBBlock` carves a region (up to 32MB) from `_spareMemory` to replenish the bucket system, builds an `Empty` entry, and advances the spare pointer
### Alignment
- Default `_align = 16`; `AlignOfLevel(level)` returns `min(levelSizeArray[level], _align)`
- Internal blocks align to level boundaries by design of slicing/edge calculations
### Complexity and Invariants
- Allocate/free are O(1) on average with bounded levels (��10) and constant-time bucket ops
- Invariants guarded by exceptions (e.g., `HLSFPositionChainBreakException`) and pointer consistency checks
## Profiling (hotspots, current workloads)
- SliceMemory ~22% (self ~17%) �� slicing and bucket enqueue dominate allocation path
- MergeAndRemoveFromBuckets ~13% (self ~12%) �� merge + removal dominate free path
- Collect ~7.6% (self ~7.5%) �� low overall cost
# HLSF (Hybrid Layer Segregated Fit) Allocator
A constant-time (O(1)) allocator for real-time, high-throughput scenarios. It uses layered size classes with segmented buckets and stores all metadata externally for clean user memory and better cache locality.
## Key Points
- O(1) allocate/free with 10 levels: 64B x 16MB; each level split into 1x/2x/3x segments
- External metadata (`Entry`, `BucketNode`) in dedicated SLABs; user blocks have no headers
- Configurable alignment (>=16B); maximum block size is configurable at construction time via the `maxBlockSize` parameter (default 32MB, 10 levels). The level array and bucket table are dynamically extended when a larger `maxBlockSize` is specified. There is no implicit fallback to `NativeMemory` for oversized requests.
- Single-threaded implementation (external sync required for multi-threaded use)
## Quick Use
```csharp
using var hlsf = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024);
var h = hlsf.Alloc(1024);
Span<byte> s = h.BufferHandle; s[0] = 42;
hlsf.Free(h);
```
## Core APIs
### Alloc(uint size)
- Purpose: Allocate a block of at least `size` bytes.
- Input: `size` in bytes. Must not exceed the `maxBlockSize` configured at construction time (default 32MB). If `size > _maxSize`, an `InvalidOperationException` is thrown.
- Output: `HybridLayerSegregatedFitAllocationHandle` with pointer access and `Span<byte>`.
- Behavior:
- Classify into level/segment and pop a matching bucket; escalate segments and levels if empty.
- Fallback to overflow bucket or carve 32MB from spare memory if all buckets are empty.
- Mark selected `Entry` as Alloc; slice any remainder into `Empty` entries and push to buckets.
- There is no implicit fallback to `NativeMemory.AlignedAlloc` for oversized requests — all allocations are served from the pre-allocated arena. The level array and bucket table are dynamically sized based on the `maxBlockSize` parameter.
### Free(HybridLayerSegregatedFitAllocationHandle handle)
- Purpose: Free a previously allocated block.
- Behavior:
- Mark the corresponding `Entry` as `Empty`.
- Merge with adjacent `Empty` entries in both directions (boundary and level constraints).
- Remove merged neighbors from buckets, update position chain, and optionally promote level.
- Re-enqueue the final `Entry` into the appropriate bucket.
### Collect()
- Purpose: Reclaim and rebalance free entries without touching allocated blocks.
- Behavior:
- Consolidate fragmented regions where possible and improve bucket coverage.
- Intended to be cheap; safe to call between benchmark phases.
### BenchmarkDotNet (HLSF vs Libc)
Environment: .NET 8.0.23 (Release), x64 RyuJIT AVX2, Windows 11, AMD Ryzen 7 6800H.
| Method | Mean | StdDev | Ratio | Allocated |
|------- |-----:|------:|------:|----------:|
| Hlsf | 838.4 µs | 40.6 µs | 1.00 | 64 B |
| Libc | 849.8 µs | 100.2 µs | 1.01 | 64 B |
Interpretation: HLSF matches libc allocation throughput under the same workload, with slightly tighter dispersion in this run.
> Note: BenchmarkDotNet also warned that the minimum iteration time was under 1ms; for more stable statistics, increase per-iteration work (e.g., higher `AllocationCount`).
## Detailed Design (summary)
-
- Size classes: The default level size array is [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216] (10 levels, 64B × 4^level). When `maxBlockSize` is specified at construction and exceeds the default range, additional levels are appended (each 4× the previous) to cover the requested maximum block size.
- Segments per level: 1x, 2x, 3x
- Buckets: index = 3 * level + segment (overflow bucket for level > 9)
- Metadata: Entry (external SLAB, position chain), BucketNode (user region, bucket chain)
- Position chain: circular doubly linked list anchored by spare sentinel
- Allocation: search buckets by level/segment with escalation; if none, carve 32MB from spare; set Alloc; slice remainder into buckets
- Slicing: align to level boundaries, pack contiguous blocks forward/backward to minimize overhead
- Free/Merge: bidirectional merge under boundary and level constraints; update buckets; optional level promotion
- Alignment: default 16B; internal blocks aligned by slicing/edge calculation
- Complexity: average O(1) with bounded levels and constant-time bucket ops
- `Entry` (external SLAB): linkage in position chain, `Position`, `Size`, `Level`, `State`
- Invariants: near neighbors must form a consistent double-linked ring; `Level` setter validates size vs. level window
- `BucketNode` (in user region): `EntryRef`, `NextLevel`, `PreviousLevel`, `IsInBucket`
- Position chain: circular doubly linked list anchored by sentinel `_spareMemory` (`State.Spare`)
### Allocation Flow
1) Compute `level` and `segment` from requested `size`
2) Try pop bucket (`TryPopLevelForAllocation(level, segment, out entry)`) and escalate:
- Advance `segment` 0��1��2, then `level++`, until `MaxLevel`
- If still empty: try pop from overflow bucket (L=10, seg=1) or `TryAllocateNew32MBBlock`
3) On success: bound-check the slice offset; set `entry` to `Alloc` using `SetEntryLevel`
4) If remainder exists in the source region, call `SliceMemory(entry->NextNear, remainder)` to produce free chunks into buckets
### Slicing Logic (`SliceMemory`)
- Inputs: `next` entry position, `sizeToSlice`
- Compute `offset` and `end` relative to base buffer; pick starting level by trailing-zero count of `offset`:
- `i = (BitOperations.TrailingZeroCount(offset) - 6) / 2`, clamped to `�� MaxLevel`
- Three phases to cover [offset, end):
1) Forward levels i..N-1: for each level size `size` and `upper=size*4`, align to `edge=((offset+upper-1)/upper)*upper`, pack `count = ((min(edge,end) - offset) / size)` blocks
2) If `i > MaxLevel`: pack in largest upper window on tail
3) Backward i-1..0: fill remaining tail with smaller sizes
- For each packed region: allocate new `Entry`, insert before `next`, mark `Empty`, and push corresponding `BucketNode` into the `(level, segment)` bucket (`PushBucketToLevel(i, count-1, b)`)
### Free and Merge (`MergeAndRemoveFromBuckets`)
- Given an `Entry` to free (turned `Empty` by caller):
- Merge left while neighbor `p` is `Empty`, same `Level`, and within boundary (`AssertBoundary`)
- Remove `p` from bucket (`RemoveFromBucket`), expand current, fix `Position`, unlink `p`
- Merge right with neighbor `n` by the same rule
- Optionally promote `Level` if `Size �� levelSizeArray[originLevel] * 4`
- Ensure `BucketNode.EntryRef` points to the merged entry; caller re-enqueues into the proper bucket
### Buckets
- `PushBucketToLevel(level, segment, node)`: push to bucket head, set `IsInBucket`, maintain prev/next
- `TryPopLevel(level, segment, out node)`: pop head; if empty, return false
- `RemoveFromBucket(level, segment, node)`: unlink from middle or use `TryPopLevel` if it is the head
### Large Allocations and Spare Memory
- Requests exceeding the configured `maxBlockSize` are rejected with `InvalidOperationException`. All allocations are served from the pre-allocated arena.
- When all buckets are empty, `TryAllocateNew32MBBlock` carves a region (up to 32MB) from `_spareMemory` to replenish the bucket system, builds an `Empty` entry, and advances the spare pointer
### Alignment
- Default `_align = 16`; `AlignOfLevel(level)` returns `min(levelSizeArray[level], _align)`
- Internal blocks align to level boundaries by design of slicing/edge calculations
### Complexity and Invariants
- Allocate/free are O(1) on average with bounded levels (��10) and constant-time bucket ops
- Invariants guarded by exceptions (e.g., `HLSFPositionChainBreakException`) and pointer consistency checks
## Profiling (hotspots, current workloads)
- SliceMemory ~22% (self ~17%) �� slicing and bucket enqueue dominate allocation path
- MergeAndRemoveFromBuckets ~13% (self ~12%) �� merge + removal dominate free path
- Collect ~7.6% (self ~7.5%) �� low overall cost
Focus: reduce per-slice math and bucket ops; add fast-paths for aligned regions; batch/lazy removals after merges.
StgSharp.Common/HighPerformance/Memory/SlabAllocationHandle.cs
+56 -56
View File
@@ -1,56 +1,56 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe struct SlabAllocationHandle<T> : IDisposable where T: unmanaged
{
private readonly SlabAllocator<T> _allocator;
internal SlabAllocationHandle(SlabAllocator<T> allocator, bool locked)
{
_allocator = allocator;
Locked = locked;
}
public bool Locked { get; set; }
public void Dispose()
{
if (Locked)
{
_allocator.ExitBufferReading();
Locked = false;
}
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.Mathematics.Memory
{
public unsafe struct SlabAllocationHandle<T> : IDisposable where T: unmanaged
{
private readonly SlabAllocator<T> _allocator;
internal SlabAllocationHandle(SlabAllocator<T> allocator, bool locked)
{
_allocator = allocator;
Locked = locked;
}
public bool Locked { get; set; }
public void Dispose()
{
if (Locked)
{
_allocator.ExitBufferReading();
Locked = false;
}
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.Chunked.cs
+134 -134
View File
@@ -1,134 +1,134 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.Chunked"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private bool _disposed;
private BufferStack<nuint> _buffers;
private BufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_buffers.Push(_currentBuffer);
_recycle = BufferStackBuilder.Create(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
// nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _currentCapacity)
{
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(newBuffer);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
// NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer))
{
NativeMemory.Free((void*)buffer);
}
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ChunkedSlabAllocator()
{
Dispose();
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.Chunked"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private bool _disposed;
private BufferStack<nuint> _buffers;
private BufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_buffers.Push(_currentBuffer);
_recycle = BufferStackBuilder.Create(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
// nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _currentCapacity)
{
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(newBuffer);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
// NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer))
{
NativeMemory.Free((void*)buffer);
}
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ChunkedSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.ChunkedConcurrent.cs
+157 -157
View File
@@ -1,157 +1,157 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.ChunkedConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private ConcurrentBufferStack<nuint> _buffers;
private ConcurrentBufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ConcurrentChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_recycle = BufferStackBuilder.CreateConcurrent(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
while (true)
{
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
nuint curIndex = Volatile.Read(ref _currentIndex);
_lock.EnterMetaDataRead();
try
{
if (curIndex >= _currentCapacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
nuint cap = Volatile.Read(ref _currentCapacity);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
_ = Interlocked.Exchange(ref _currentIndex, 0);
}
finally
{
_lock.ExitExpansionProcess();
}
}
curIndex = Volatile.Read(ref _currentIndex);
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer)) {
NativeMemory.Free((void*)buffer);
}
_lock.Dispose();
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ConcurrentChunkedSlabAllocator()
{
Dispose();
}
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.ChunkedConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private ConcurrentBufferStack<nuint> _buffers;
private ConcurrentBufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ConcurrentChunkedSlabAllocator(nuint count)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_recycle = BufferStackBuilder.CreateConcurrent(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
while (true)
{
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
nuint curIndex = Volatile.Read(ref _currentIndex);
_lock.EnterMetaDataRead();
try
{
if (curIndex >= _currentCapacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
nuint cap = Volatile.Read(ref _currentCapacity);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
_ = Interlocked.Exchange(ref _currentIndex, 0);
}
finally
{
_lock.ExitExpansionProcess();
}
}
curIndex = Volatile.Read(ref _currentIndex);
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer)) {
NativeMemory.Free((void*)buffer);
}
_lock.Dispose();
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ConcurrentChunkedSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.SequencialConcurrent.cs
+164 -164
View File
@@ -1,165 +1,165 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.SequencialConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private T* _buffer;
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private readonly ConcurrentBufferStack<nuint> _stack;
private readonly int _elementSize;
private nuint _highestAllocated;
private ulong _capacity;
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
{
_elementSize = Unsafe.SizeOf<T>();
_capacity = initialCapacity;
_highestAllocated = 0;
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
int initCount = _capacity switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)_capacity / 4
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = (nuint)i;
}
_stack = BufferStackBuilder.CreateConcurrent(span);
_highestAllocated += (nuint)initCount;
}
internal T* BasePointer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _buffer;
}
public override nuint Allocate()
{
if (_stack.TryPop(out nuint index)) {
return index;
}
while (true)
{
index = Volatile.Read(ref _highestAllocated);
_lock.EnterMetaDataRead();
try
{
if (index >= _capacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
_capacity = Volatile.Read(ref _capacity) * 2;
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
_lock.EnterBufferCopy();
try
{
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
return index;
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_stack.Dispose();
NativeMemory.Free(_buffer);
_lock.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading()
{
_lock.EnterBufferRead();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading()
{
_lock.ExitBufferRead();
}
public override void Free(nuint index)
{
_stack.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
EnterBufferReading();
return new SlabAllocationHandle<T>(this, true);
}
~ConcurrentSequentialSlabAllocator()
{
Dispose();
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.SequencialConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
{
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
private T* _buffer;
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private readonly ConcurrentBufferStack<nuint> _stack;
private readonly int _elementSize;
private nuint _highestAllocated;
private ulong _capacity;
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
{
_elementSize = Unsafe.SizeOf<T>();
_capacity = initialCapacity;
_highestAllocated = 0;
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
int initCount = _capacity switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)_capacity / 4
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = (nuint)i;
}
_stack = BufferStackBuilder.CreateConcurrent(span);
_highestAllocated += (nuint)initCount;
}
internal T* BasePointer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _buffer;
}
public override nuint Allocate()
{
if (_stack.TryPop(out nuint index)) {
return index;
}
while (true)
{
index = Volatile.Read(ref _highestAllocated);
_lock.EnterMetaDataRead();
try
{
if (index >= _capacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
_capacity = Volatile.Read(ref _capacity) * 2;
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
_lock.EnterBufferCopy();
try
{
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
return index;
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_stack.Dispose();
NativeMemory.Free(_buffer);
_lock.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading()
{
_lock.EnterBufferRead();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading()
{
_lock.ExitBufferRead();
}
public override void Free(nuint index)
{
_stack.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
EnterBufferReading();
return new SlabAllocationHandle<T>(this, true);
}
~ConcurrentSequentialSlabAllocator()
{
Dispose();
}
}
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs
+118 -118
View File
@@ -1,118 +1,118 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
namespace StgSharp.Mathematics.Memory
{
public abstract class SlabAllocator<T> : IDisposable where T: unmanaged
{
public abstract nuint Allocate();
/// <summary>
/// Creates a new SLAB allocator instance with the specified configuration.
/// </summary>
/// <param name="count">
/// Initial number of objects that can be allocated.
/// </param>
/// <param name="layout">
/// Memory layout strategy for the SLAB allocator.
/// </param>
/// <param name="concurrentSupport">
/// Whether to enable concurrent access support.
/// </param>
/// <returns>
/// A new SLAB allocator instance.
/// </returns>
/// <exception cref="ArgumentOutOfRangeException">
/// Thrown when an invalid layout is specified.
/// </exception>
/// <exception cref="NotSupportedException">
/// Thrown when requesting unsupported configurations.
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
/// /// allocators themselves may be designed for single-threaded use, allocated objects
/// are /// frequently shared across multiple threads in real-world applications. Due to
/// the /// special implementation of expandable concurrent sequential SLAB allocators,
/// they rely /// on internal expansion locks to ensure that memory buffer expansions
/// (which involve /// copying the entire buffer to a new location) do not occur while
/// other threads are /// reading from allocated regions. </para> <para><strong>
/// Sequential Layout Limitations: ///</strong><br /> Sequential layout SLAB allocators
/// cannot provide a single-threaded /// version because:<list
/// type="bullet"><item><description>They use contiguous memory /// layout that requires
/// buffer expansion when capacity is exceeded
/// ///</description></item><item><description>Buffer expansion involves copying all ///
/// existing data to a new memory location</description></item><item><description> ///
/// Without proper synchronization, concurrent access to allocated objects during ///
/// expansion would cause memory corruption</description></item><item><description>The
/// /// expansion lock mechanism is essential even if allocation itself is single- ///
/// threaded </description></item></list></para> <para><strong>Chunked Layout:
/// ///</strong><br /> Chunked layout allocators can optionally provide single-threaded
/// /// versions since they don't require buffer expansion and copying. Each chunk is
/// /// allocated independently, avoiding the memory relocation issues present in
/// sequential /// layouts. </para>
/// </remarks>
public static SlabAllocator<T> Create(nuint count, SlabBufferLayout layout, bool concurrentSupport = true)
{
if (concurrentSupport) {
return layout switch
{
SlabBufferLayout.Sequential => new ConcurrentSequentialSlabAllocator<T>(count),
SlabBufferLayout.Chunked => new ConcurrentChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
return layout switch
{
SlabBufferLayout.Sequential => throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."),
SlabBufferLayout.Chunked => new ChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
public abstract void Dispose();
public abstract void EnterBufferReading();
public abstract void ExitBufferReading();
public abstract void Free(nuint index);
public abstract SlabAllocationHandle<T> ReadAllocation();
}
public enum SlabBufferLayout
{
Sequential,
Chunked
}
}
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
namespace StgSharp.Mathematics.Memory
{
public abstract class SlabAllocator<T> : IDisposable where T: unmanaged
{
public abstract nuint Allocate();
/// <summary>
/// Creates a new SLAB allocator instance with the specified configuration.
/// </summary>
/// <param name="count">
/// Initial number of objects that can be allocated.
/// </param>
/// <param name="layout">
/// Memory layout strategy for the SLAB allocator.
/// </param>
/// <param name="concurrentSupport">
/// Whether to enable concurrent access support.
/// </param>
/// <returns>
/// A new SLAB allocator instance.
/// </returns>
/// <exception cref="ArgumentOutOfRangeException">
/// Thrown when an invalid layout is specified.
/// </exception>
/// <exception cref="NotSupportedException">
/// Thrown when requesting unsupported configurations.
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
/// /// allocators themselves may be designed for single-threaded use, allocated objects
/// are /// frequently shared across multiple threads in real-world applications. Due to
/// the /// special implementation of expandable concurrent sequential SLAB allocators,
/// they rely /// on internal expansion locks to ensure that memory buffer expansions
/// (which involve /// copying the entire buffer to a new location) do not occur while
/// other threads are /// reading from allocated regions. </para> <para><strong>
/// Sequential Layout Limitations: ///</strong><br /> Sequential layout SLAB allocators
/// cannot provide a single-threaded /// version because:<list
/// type="bullet"><item><description>They use contiguous memory /// layout that requires
/// buffer expansion when capacity is exceeded
/// ///</description></item><item><description>Buffer expansion involves copying all ///
/// existing data to a new memory location</description></item><item><description> ///
/// Without proper synchronization, concurrent access to allocated objects during ///
/// expansion would cause memory corruption</description></item><item><description>The
/// /// expansion lock mechanism is essential even if allocation itself is single- ///
/// threaded </description></item></list></para> <para><strong>Chunked Layout:
/// ///</strong><br /> Chunked layout allocators can optionally provide single-threaded
/// /// versions since they don't require buffer expansion and copying. Each chunk is
/// /// allocated independently, avoiding the memory relocation issues present in
/// sequential /// layouts. </para>
/// </remarks>
public static SlabAllocator<T> Create(nuint count, SlabBufferLayout layout, bool concurrentSupport = true)
{
if (concurrentSupport) {
return layout switch
{
SlabBufferLayout.Sequential => new ConcurrentSequentialSlabAllocator<T>(count),
SlabBufferLayout.Chunked => new ConcurrentChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
return layout switch
{
SlabBufferLayout.Sequential => throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."),
SlabBufferLayout.Chunked => new ChunkedSlabAllocator<T>(count),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
public abstract void Dispose();
public abstract void EnterBufferReading();
public abstract void ExitBufferReading();
public abstract void Free(nuint index);
public abstract SlabAllocationHandle<T> ReadAllocation();
}
public enum SlabBufferLayout
{
Sequential,
Chunked
}
}