clear merging conflict
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
//-----------------------------------------------------------------------
|
||||
//-----------------------------------------------------------------------
|
||||
// file="IRegisterType.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
//-----------------------------------------------------------------------
|
||||
//-----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public interface IRegisterType
|
||||
{
|
||||
|
||||
public ref T AsRef<T>() where T: unmanaged, INumber<T>;
|
||||
|
||||
public ref T Member<T>(int index) where T: unmanaged, INumber<T>;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="BufferStack.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
|
||||
public unsafe class BufferStack<T> : IDisposable where T: unmanaged
|
||||
{
|
||||
|
||||
private byte* _buffer;
|
||||
private ulong _count;
|
||||
|
||||
public BufferStack(int capacity)
|
||||
{
|
||||
if (capacity <= 0) {
|
||||
throw new ArgumentOutOfRangeException(nameof(capacity));
|
||||
}
|
||||
|
||||
AllocateBuffer(capacity);
|
||||
}
|
||||
|
||||
public int Count
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (int)_count;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Clear()
|
||||
{
|
||||
((Header*)_buffer)->Top = 0;
|
||||
_count = 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Dispose()
|
||||
{
|
||||
Marshal.FreeHGlobal((nint)_buffer);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T Pop()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top == 0) {
|
||||
throw new InvalidOperationException("Stack underflow");
|
||||
}
|
||||
|
||||
header->Top--;
|
||||
_count--;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
return dataStart[header->Top];
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Push(T item)
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top >= header->Capacity) {
|
||||
Expand();
|
||||
}
|
||||
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
dataStart[header->Top] = item;
|
||||
header->Top++;
|
||||
_count++;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryPop(out T value)
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top == 0)
|
||||
{
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
header->Top--;
|
||||
_count--;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
value = dataStart[header->Top];
|
||||
return true;
|
||||
}
|
||||
|
||||
// Internal method for efficient bulk initialization
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
|
||||
{
|
||||
if (span.IsEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
Header* header = (Header*)_buffer;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
|
||||
fixed (T* sourcePtr = span) {
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart,
|
||||
header->Capacity * (nuint)sizeof(T),
|
||||
(nuint)span.Length * (nuint)sizeof(T));
|
||||
}
|
||||
|
||||
header->Top = (nuint)span.Length;
|
||||
_count = (ulong)span.Length;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AllocateBuffer(int capacity)
|
||||
{
|
||||
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
|
||||
_buffer = (byte*)NativeMemory.Alloc(size);
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
header->Capacity = (nuint)capacity;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void Expand()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
nuint newCap = header->Capacity * 2;
|
||||
nuint newSize = (nuint)sizeof(Header) + newCap * (nuint)sizeof(T);
|
||||
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
|
||||
header = (Header*)_buffer;
|
||||
header->Capacity = newCap;
|
||||
}
|
||||
|
||||
private struct Header
|
||||
{
|
||||
|
||||
public nuint Capacity;
|
||||
public nuint Top;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="ConcurrentBufferStack.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public static class BufferStackBuilder
|
||||
{
|
||||
|
||||
// For BufferStack collection builder
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static BufferStack<T> Create<T>(ReadOnlySpan<T> span) where T: unmanaged
|
||||
{
|
||||
if (span.IsEmpty)
|
||||
{
|
||||
return new BufferStack<T>(4); // Default small capacity
|
||||
}
|
||||
|
||||
BufferStack<T> stack = new BufferStack<T>(span.Length);
|
||||
|
||||
|
||||
// Use internal method for efficient bulk initialization
|
||||
stack.InitializeFromSpan(span);
|
||||
|
||||
return stack;
|
||||
}
|
||||
|
||||
// For ConcurrentBufferStack collection builder
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static ConcurrentBufferStack<T> CreateConcurrent<T>(ReadOnlySpan<T> span) where T: unmanaged
|
||||
{
|
||||
if (span.IsEmpty)
|
||||
{
|
||||
return new ConcurrentBufferStack<T>(4); // Default small capacity
|
||||
}
|
||||
|
||||
ConcurrentBufferStack<T> stack = new ConcurrentBufferStack<T>(span.Length);
|
||||
|
||||
|
||||
// Use internal method for efficient bulk initialization
|
||||
stack.InitializeFromSpan(span);
|
||||
|
||||
return stack;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[CollectionBuilder(builderType: typeof(BufferStackBuilder), methodName: nameof(BufferStackBuilder.CreateConcurrent))]
|
||||
public unsafe class ConcurrentBufferStack<T> : IDisposable where T: unmanaged
|
||||
{
|
||||
|
||||
private byte* _buffer;
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private ulong _count;
|
||||
|
||||
public ConcurrentBufferStack(int capacity)
|
||||
{
|
||||
if (capacity <= 0) {
|
||||
throw new ArgumentOutOfRangeException(nameof(capacity));
|
||||
}
|
||||
|
||||
AllocateBuffer(capacity);
|
||||
}
|
||||
|
||||
public ulong Count
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => Volatile.Read(ref _count);
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
header->Top = 0;
|
||||
Volatile.Write(ref _count, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_lock.Dispose();
|
||||
Marshal.FreeHGlobal((nint)_buffer);
|
||||
}
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
while (true)
|
||||
{
|
||||
nuint oldTop = Volatile.Read(ref header->Top);
|
||||
if (oldTop <= 0) {
|
||||
throw new InvalidOperationException("Stack underflow");
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
try
|
||||
{
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
_ = Interlocked.Decrement(ref _count);
|
||||
return dataStart[oldTop - 1];
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
}
|
||||
Thread.Sleep(0); // Yield to avoid busy waiting
|
||||
}
|
||||
}
|
||||
|
||||
public void Push(T item)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
nuint oldTop = Volatile.Read(ref header->Top);
|
||||
nuint cap = Volatile.Read(ref header->Capacity);
|
||||
if (oldTop >= cap)
|
||||
{
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
header = (Header*)_buffer;
|
||||
if (header->Top >= header->Capacity)
|
||||
{
|
||||
nuint newCap = header->Capacity * 2;
|
||||
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
|
||||
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
|
||||
((Header*)_buffer)->Capacity = newCap;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref header->Top, oldTop + 1, oldTop) == oldTop)
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
try
|
||||
{
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
dataStart[oldTop] = item;
|
||||
_ = Interlocked.Increment(ref _count);
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
Thread.Sleep(0); // Yield to avoid busy waiting
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryPop(out T value)
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
while (true)
|
||||
{
|
||||
nuint oldTop = Volatile.Read(ref header->Top);
|
||||
if (oldTop <= 0)
|
||||
{
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
try
|
||||
{
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
value = dataStart[oldTop - 1];
|
||||
_ = Interlocked.Decrement(ref _count);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Thread.Sleep(0); // Yield to avoid busy waiting
|
||||
}
|
||||
}
|
||||
|
||||
// Internal method for efficient bulk initialization
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal unsafe void InitializeFromSpan(ReadOnlySpan<T> span)
|
||||
{
|
||||
if (span.IsEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
Header* header = (Header*)_buffer;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
|
||||
fixed (T* sourcePtr = span) {
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart,
|
||||
header->Capacity * (nuint)sizeof(T),
|
||||
(nuint)span.Length * (nuint)sizeof(T));
|
||||
}
|
||||
|
||||
header->Top = (nuint)span.Length;
|
||||
Volatile.Write(ref _count, (ulong)span.Length);
|
||||
}
|
||||
|
||||
private void AllocateBuffer(int capacity)
|
||||
{
|
||||
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
|
||||
_buffer = (byte*)NativeMemory.Alloc(size);
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
header->Capacity = (nuint)capacity;
|
||||
}
|
||||
|
||||
private struct Header
|
||||
{
|
||||
|
||||
public nuint Capacity; // max capacity of the stack (number of elements)
|
||||
public nuint Top; // next available index in the stack
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -61,6 +61,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
|
||||
fallback:
|
||||
i = level;
|
||||
s = segmentIndex;
|
||||
while (!TryPopLevelForAllocation(i, s, out handle))
|
||||
@@ -76,7 +77,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
|
||||
}
|
||||
|
||||
break;
|
||||
Thread.Sleep(0);
|
||||
goto fallback;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,23 +86,26 @@ namespace StgSharp.HighPerformance.Memory
|
||||
SliceMemory(handle, (segmentIndex + 1) * levelSizeArray[level]);
|
||||
}
|
||||
|
||||
return new hlsfHandle(handle, size);
|
||||
return new hlsfHandle(handle, size);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntryNextLock(handle);
|
||||
}
|
||||
}
|
||||
|
||||
private void SliceMemory(Entry* e, int offset)
|
||||
{
|
||||
Entry* current = e;
|
||||
ulong n = e->NextNear;
|
||||
Entry* next = (Entry*)n;
|
||||
Entry* next = e;
|
||||
|
||||
ulong originOffset = e->Position - (ulong)m_Buffer;
|
||||
ulong finalOffset = originOffset + (ulong)offset;
|
||||
ulong end = e->Position + e->Size - (ulong)m_Buffer;
|
||||
int remainSize = (int)(e->Size - offset);
|
||||
|
||||
if ((originOffset & 63) != 0) {
|
||||
throw new InvalidOperationException("Entry position offset is not aligned to 64 bytes.");
|
||||
}
|
||||
AcquireEntryNextLock(e);
|
||||
AcquireEntryPrevLock(next);
|
||||
|
||||
int minLevel = Math.Max((BitOperations.TrailingZeroCount(finalOffset) - 6) / 2, 0);
|
||||
int i = minLevel >= levelSizeArray.Length ? levelSizeArray.Length - 1 : minLevel;
|
||||
@@ -112,9 +117,13 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int blockCount = (int)((ulong.Min(board, end) - board) / (ulong)levelSize);
|
||||
if (blockCount == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
int s = blockCount * levelSize;
|
||||
int levelSize = levelSizeArray[i];
|
||||
int blockCount = remainSize / levelSize % 4;
|
||||
if (blockCount == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
int s = blockCount * levelSize;
|
||||
|
||||
// create and init new entry
|
||||
Entry* newEntry = (Entry*)_entries.Allocate();
|
||||
@@ -123,12 +132,24 @@ namespace StgSharp.HighPerformance.Memory
|
||||
BucketNode* bucket = (BucketNode*)newEntry->Position;
|
||||
bucket->EntryRef = newEntry;
|
||||
|
||||
// link new entry
|
||||
newEntry->PreviousNear = (ulong)current;
|
||||
newEntry->NextNear = n;
|
||||
current->NextNear = (ulong)newEntry;
|
||||
if (next != null) {
|
||||
// link new entry
|
||||
newEntry->PreviousNear = (ulong)current;
|
||||
newEntry->NextNear = n;
|
||||
current->NextNear = (ulong)newEntry;
|
||||
next->PreviousNear = (ulong)newEntry;
|
||||
|
||||
newEntry->Level = i;
|
||||
PushLevel(i, blockCount - 1, bucket);
|
||||
|
||||
// set new entry
|
||||
Entry* old = current;
|
||||
current = newEntry;
|
||||
|
||||
// update lock and data
|
||||
finalOffset += (ulong)s;
|
||||
remainSize -= s;
|
||||
ReleaseEntryNextLock(old);
|
||||
AcquireEntryNextLock(current);
|
||||
}
|
||||
|
||||
newEntry->Level = i;
|
||||
@@ -183,8 +204,63 @@ namespace StgSharp.HighPerformance.Memory
|
||||
remainSize -= s;
|
||||
if (remainSize <= 0)
|
||||
{
|
||||
break;
|
||||
ReleaseEntryNextLock(current);
|
||||
ReleaseEntryPrevLock(next);
|
||||
return;
|
||||
}
|
||||
for (; i >= 0; i--)
|
||||
{
|
||||
int levelSize = levelSizeArray[i];
|
||||
int blockCount = remainSize / levelSize % 4;
|
||||
if (blockCount == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
int s = blockCount * levelSize;
|
||||
|
||||
// create and init new entry
|
||||
Entry* newEntry = (Entry*)_entries.Allocate();
|
||||
newEntry->LockBuffer = 0;
|
||||
newEntry->Position = (nuint)((ulong)m_Buffer + finalOffset);
|
||||
newEntry->Size = (uint)s;
|
||||
BucketNode* bucket = (BucketNode*)newEntry->Position;
|
||||
bucket->EntryRef = newEntry;
|
||||
bucket->LockBuffer = 0;
|
||||
newEntry->State = EntryState.ThreadOccupied;
|
||||
|
||||
// link new entry
|
||||
newEntry->PreviousNear = (ulong)current;
|
||||
current->NextNear = (ulong)newEntry;
|
||||
newEntry->NextNear = n;
|
||||
next->PreviousNear = (ulong)newEntry;
|
||||
|
||||
newEntry->Level = i;
|
||||
PushLevel(i, blockCount - 1, bucket);
|
||||
|
||||
// set new entry
|
||||
Entry* old = current;
|
||||
current = newEntry;
|
||||
|
||||
// update lock and data
|
||||
finalOffset += (ulong)s;
|
||||
remainSize -= s;
|
||||
ReleaseEntryNextLock(old);
|
||||
AcquireEntryNextLock(current);
|
||||
if (remainSize <= 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Release locks in reverse order
|
||||
ReleaseEntryNextLock(current);
|
||||
ReleaseEntryPrevLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,13 +275,22 @@ namespace StgSharp.HighPerformance.Memory
|
||||
/// </returns>
|
||||
private bool TryAllocateNew16MBBlock(out Entry* handle)
|
||||
{
|
||||
nuint pos = _spareMemory->Position;
|
||||
nuint pos = Volatile.Read(ref _spareMemory->Position);
|
||||
nuint blockSize = (nuint)levelSizeArray[^1] * 2; // Use last level size (16MB)
|
||||
nuint newPos = pos + blockSize;
|
||||
|
||||
ulong p = _spareMemory->PreviousNear;
|
||||
Entry* previous = (Entry*)p;
|
||||
|
||||
ulong p;
|
||||
Entry* previous;
|
||||
do
|
||||
{
|
||||
p = Volatile.Read(ref _spareMemory->PreviousNear);
|
||||
previous = (Entry*)p;
|
||||
if (previous != null) {
|
||||
AcquireEntryNextLock(previous);
|
||||
}
|
||||
AcquireEntryPrevLock(_spareMemory);
|
||||
break;
|
||||
} while (true);
|
||||
if (_spareMemory->Size <= 0)
|
||||
{
|
||||
handle = null;
|
||||
@@ -225,20 +310,34 @@ namespace StgSharp.HighPerformance.Memory
|
||||
handle->Size = (uint)blockSize;
|
||||
handle->Level = GetLevelFromSize((uint)blockSize);
|
||||
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
BucketNode* b = (BucketNode*)handle->Position;
|
||||
b->EntryRef = handle;
|
||||
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
|
||||
handle->PrevLock = 0;
|
||||
handle->NextLock = 0;
|
||||
|
||||
// update position linked list
|
||||
handle->NextNear = (ulong)_spareMemory;
|
||||
handle->PreviousNear = p;
|
||||
_spareMemory->PreviousNear = (ulong)handle;
|
||||
if (previous != null) {
|
||||
previous->NextNear = (ulong)handle;
|
||||
// Update spare memory position
|
||||
_spareMemory->Position = newPos;
|
||||
BucketNode* b = (BucketNode*)handle->Position;
|
||||
b->EntryRef = handle;
|
||||
PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
|
||||
|
||||
// update position linked list
|
||||
handle->NextNear = (ulong)_spareMemory;
|
||||
handle->PreviousNear = p;
|
||||
_spareMemory->PreviousNear = (ulong)handle;
|
||||
if (previous != null) {
|
||||
previous->NextNear = (ulong)handle;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseEntryNextLock(previous);
|
||||
ReleaseEntryPrevLock(_spareMemory);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -39,6 +39,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
AcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize bucket system (called in constructor)
|
||||
/// </summary>
|
||||
@@ -47,16 +53,6 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_bucketHeads = new ulong[levelSizeArray.Length * 3];
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
private void PushLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
@@ -64,112 +60,269 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if (_bucketHeads[index] == EmptyHandle)
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, EmptyHandle) == EmptyHandle)
|
||||
{
|
||||
_bucketHeads[index] = (nuint)node;
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return;
|
||||
} else
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
AcquireBucketNodeNextLock(node);
|
||||
ulong head = Volatile.Read(ref _bucketHeads[index]);
|
||||
node->NextLevel = head;
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], (nuint)node, head) == head)
|
||||
{
|
||||
if (headNode != null && !TryAcquireBucketNodePrevLock(headNode))
|
||||
{
|
||||
// reset state and retry
|
||||
_ = Interlocked.CompareExchange(ref _bucketHeads[index], head, (nuint)node);
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
}
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (headNode != null) {
|
||||
ReleaseBucketNodePrevLock(headNode);
|
||||
}
|
||||
}
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Add to head of linked list
|
||||
ulong head = _bucketHeads[index];
|
||||
BucketNode* headNode = (BucketNode*)head;
|
||||
|
||||
node->NextLevel = head;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = true;
|
||||
node->Magic = BucketNode.MAGIC_NUMBER;
|
||||
|
||||
if (headNode != null) {
|
||||
headNode->PreviousLevel = (ulong)node;
|
||||
/// <summary>
|
||||
/// 根据锁类型释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeLockByType(BucketNode* node, byte lockType)
|
||||
{
|
||||
switch (lockType)
|
||||
{
|
||||
case 0:
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
return; // PrevLock
|
||||
case 1:
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
return; // BothLocks
|
||||
case 2:
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
return; // NextLock
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
_bucketHeads[index] = (nuint)node;
|
||||
/// <summary>
|
||||
/// 按地址顺序释放BucketNode锁
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
if (previous != null) {
|
||||
ReleaseBucketNodeNextLock(previous);
|
||||
}
|
||||
if (node != null) {
|
||||
ReleaseBucketNodeBothLocks(node);
|
||||
}
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
|
||||
if ((BucketNode*)_bucketHeads[index] == node)
|
||||
while (true)
|
||||
{
|
||||
_bucketHeads[index] = EmptyHandle;
|
||||
return;
|
||||
}
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
|
||||
return;
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong p = node->PreviousLevel;
|
||||
ulong n = node->NextLevel;
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
if (previous != null) {
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
if (!TryAcquireOrderedBucketLocks(previous, node, next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
try
|
||||
{
|
||||
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n)
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (previous != null) {
|
||||
previous->NextLevel = n;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousLevel = p;
|
||||
}
|
||||
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
node->IsInBucket = false;
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseOrderedBucketLocks(previous, node, next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeSpinLock(BucketNode* node)
|
||||
{
|
||||
return TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireOrderedBucketLocks(BucketNode* previous, BucketNode* node, BucketNode* next)
|
||||
{
|
||||
if (previous != null && node != null && next != null)
|
||||
{
|
||||
// previous -> node -> next
|
||||
return TryAcquireBucketNodeNextLock(previous) &&
|
||||
TryAcquireBucketNodeBothLocks(node) &&
|
||||
TryAcquireBucketNodePrevLock(next);
|
||||
} else if (previous != null && node != null) {
|
||||
return TryAcquireBucketNodeNextLock(previous) && TryAcquireBucketNodeBothLocks(node);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
ulong h = _bucketHeads[index];
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
while (true)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
ulong h = Volatile.Read(ref _bucketHeads[index]);
|
||||
BucketNode* head = (BucketNode*)h;
|
||||
|
||||
if (head == null)
|
||||
{
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!TryAcquireBucketNodeNextLock(head))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
ulong n = Volatile.Read(ref head->NextLevel);
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
if (next == null)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], EmptyHandle, h) == h)
|
||||
{
|
||||
node = head;
|
||||
return true;
|
||||
}
|
||||
node = null;
|
||||
return false;
|
||||
}
|
||||
try
|
||||
{
|
||||
if (!TryAcquireBucketNodePrevLock(next))
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _bucketHeads[index], head->NextLevel, h) == h)
|
||||
{
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
Thread.Sleep(0);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (next != null) {
|
||||
ReleaseBucketNodePrevLock(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
ReleaseBucketNodeNextLock(head);
|
||||
}
|
||||
}
|
||||
|
||||
ulong n = head->NextLevel;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
if (next == null)
|
||||
{
|
||||
_bucketHeads[index] = EmptyHandle;
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
node->PreviousLevel = EmptyHandle;
|
||||
return true;
|
||||
}
|
||||
|
||||
_bucketHeads[index] = head->NextLevel;
|
||||
node = head;
|
||||
node->IsInBucket = false;
|
||||
node->NextLevel = EmptyHandle;
|
||||
|
||||
if (next != null) {
|
||||
next->PreviousLevel = EmptyHandle;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
{
|
||||
int index = (3 * levelIndex) + sizeIndex;
|
||||
if ((BucketNode*)_bucketHeads[index] == node)
|
||||
if ((BucketNode*)Volatile.Read(ref _bucketHeads[index]) == node)
|
||||
{
|
||||
_bucketHeads[index] = EmptyHandle;
|
||||
return true;
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
ulong p = Volatile.Read(ref node->PreviousLevel);
|
||||
ulong n = Volatile.Read(ref node->NextLevel);
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
|
||||
if (!TryAcquireOrderedBucketLocks(previous, node, next)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
ulong p = node->PreviousLevel;
|
||||
ulong n = node->NextLevel;
|
||||
BucketNode* previous = (BucketNode*)p;
|
||||
BucketNode* next = (BucketNode*)n;
|
||||
if (Volatile.Read(ref node->PreviousLevel) != p || Volatile.Read(ref node->NextLevel) != n) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (previous != null) {
|
||||
previous->NextLevel = n;
|
||||
@@ -183,9 +336,9 @@ namespace StgSharp.HighPerformance.Memory
|
||||
node->IsInBucket = false;
|
||||
return true;
|
||||
}
|
||||
catch
|
||||
finally
|
||||
{
|
||||
return false;
|
||||
ReleaseOrderedBucketLocks(previous, node, next);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,7 +346,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
/// BucketNode structure - specialized node for bucket linked lists Stored in main
|
||||
/// allocation area (user-overwritable region)
|
||||
/// </summary>
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||
[StructLayout(LayoutKind.Explicit, Size = 40)]
|
||||
internal struct BucketNode
|
||||
{
|
||||
|
||||
@@ -201,12 +354,149 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
[FieldOffset(16)] internal Entry* EntryRef; // Pointer to corresponding Entry
|
||||
[FieldOffset(28)] internal bool IsInBucket; // Whether in bucket
|
||||
[FieldOffset(36)] internal int NextLock;
|
||||
[FieldOffset(32)] internal int PrevLock;
|
||||
[FieldOffset(24)] internal uint Magic; // Magic number verification
|
||||
[FieldOffset(32)] internal ulong LockBuffer;
|
||||
|
||||
[FieldOffset(0)] internal ulong NextLevel; // Bucket stack linked list: next pointer
|
||||
[FieldOffset(8)] internal ulong PreviousLevel; // Bucket stack linked list: previous pointer
|
||||
|
||||
}
|
||||
|
||||
#region BucketNode Operations
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsNullOrEmpty(BucketNode* node)
|
||||
{
|
||||
return node == null || node->Magic != BucketNode.MAGIC_NUMBER;
|
||||
}
|
||||
|
||||
// BucketNode 前驱锁操作(控制PreviousLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->PrevLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->PrevLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodePrevLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->PrevLock) == thread) {
|
||||
Volatile.Write(ref node->PrevLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
// BucketNode 后继锁操作(控制NextLevel字段)
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
return Interlocked.CompareExchange(ref node->NextLock, thread, 0) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) != thread)
|
||||
{
|
||||
while (Interlocked.CompareExchange(ref node->NextLock, thread, 0) != 0) {
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeNextLock(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
int thread = Environment.CurrentManagedThreadId;
|
||||
if (Volatile.Read(ref node->NextLock) == thread) {
|
||||
Volatile.Write(ref node->NextLock, 0);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (TryAcquireBucketNodePrevLock(node))
|
||||
{
|
||||
if (TryAcquireBucketNodeNextLock(node))
|
||||
{
|
||||
return true;
|
||||
} else
|
||||
{
|
||||
ReleaseBucketNodePrevLock(node); // 回滚
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseBucketNodePrevLock(node);
|
||||
ReleaseBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireBucketNodeBothLocks(BucketNode* node)
|
||||
{
|
||||
if (node == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
AcquireBucketNodePrevLock(node);
|
||||
AcquireBucketNodeNextLock(node);
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,44 +41,6 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
private object _lock = new object();
|
||||
|
||||
public void DisplayAllocatedEntryLayout(bool singleStep)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
Entry* cur = _spareMemory;
|
||||
while (cur != null)
|
||||
{
|
||||
if (cur->State == EntryState.Allocated)
|
||||
{
|
||||
Console.WriteLine(EntryToString(cur));
|
||||
if (singleStep) {
|
||||
Console.ReadKey();
|
||||
}
|
||||
}
|
||||
cur = (Entry*)cur->PreviousNear;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void DisplayEmptyEntryLayout(bool singleStep)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
Entry* cur = _spareMemory;
|
||||
while (cur != null)
|
||||
{
|
||||
if (cur->State == EntryState.Empty)
|
||||
{
|
||||
Console.WriteLine(EntryToString(cur));
|
||||
if (singleStep) {
|
||||
Console.ReadKey();
|
||||
}
|
||||
}
|
||||
cur = (Entry*)cur->PreviousNear;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void DisplayEntryLayout(bool singleStep)
|
||||
{
|
||||
lock (_lock)
|
||||
|
||||
@@ -28,6 +28,13 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
@@ -47,6 +54,13 @@ namespace StgSharp.HighPerformance.Memory
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(e);
|
||||
try
|
||||
{
|
||||
if (GetEntryState(e) != EntryState.Allocated) {
|
||||
return;
|
||||
}
|
||||
SetEntryState(e, EntryState.ThreadOccupied);
|
||||
|
||||
if (e->State != EntryState.Allocated) {
|
||||
return;
|
||||
|
||||
@@ -41,18 +41,24 @@ namespace StgSharp.HighPerformance.Memory
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void AcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
AcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
private bool AssertBackwardBoundary(ulong current, Entry* target, int level)
|
||||
{
|
||||
ulong align = (ulong)levelSizeArray[level + 1];
|
||||
ulong targetOffset = target->Position - (ulong)m_Buffer;
|
||||
return current / align == targetOffset / align;
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == (current + target->Size) >> size;
|
||||
}
|
||||
|
||||
private bool AssertForwardBoundary(ulong current, Entry* target, int level)
|
||||
{
|
||||
ulong align = (ulong)levelSizeArray[level + 1];
|
||||
ulong targetOffset = target->Position + target->Size - (ulong)m_Buffer;
|
||||
return current / align == targetOffset / align;
|
||||
int size = (level * 2) + 6;
|
||||
ulong align = current >> size;
|
||||
return align == target->Position >> size && align == (current - target->Size) >> size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -77,7 +83,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
/// <summary>
|
||||
/// Used in Free operations, efficiently remove from buckets and merge adjacent blocks.
|
||||
/// Current Entry will serve as the final merged block.
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// strategy, any fail in acquiring spin lock will end a merging process.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
{
|
||||
@@ -85,6 +92,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
return;
|
||||
}
|
||||
|
||||
Entry* result = entry;
|
||||
uint size = entry->Size;
|
||||
int originLevel = GetLevelFromSize(size);
|
||||
int segment = DetermineSegmentIndex(size, originLevel);
|
||||
@@ -154,7 +162,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
private void RecycleEntry(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
if (entry is null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -163,29 +171,45 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_entries.Free((nuint)entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void ReleaseEntrySpinLock(Entry* entry)
|
||||
{
|
||||
ReleaseEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remove node from position linked list
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
AcquireEntrySpinLock(entry);
|
||||
try
|
||||
{
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
|
||||
Entry* prev = (Entry*)entry->PreviousNear;
|
||||
Entry* next = (Entry*)entry->NextNear;
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
}
|
||||
|
||||
if (prev != null) {
|
||||
prev->NextNear = entry->NextNear;
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_entries.Free((nuint)entry);
|
||||
}
|
||||
if (next != null) {
|
||||
next->PreviousNear = entry->PreviousNear;
|
||||
finally
|
||||
{
|
||||
ReleaseEntrySpinLock(entry);
|
||||
}
|
||||
}
|
||||
|
||||
// Clear node pointers
|
||||
entry->NextNear = EmptyHandle;
|
||||
entry->PreviousNear = EmptyHandle;
|
||||
_entries.Free((nuint)entry);
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool TryAcquireEntrySpinLock(Entry* entry)
|
||||
{
|
||||
return TryAcquireEntryBothLocks(entry);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -198,6 +222,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
[FieldOffset(28)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(24)] internal uint Size; // Block size information
|
||||
[FieldOffset(32)] internal ulong LockBuffer;
|
||||
[FieldOffset(8)] internal ulong NextNear; // Position linked list: next pointer
|
||||
[FieldOffset(16)] internal ulong PreviousNear; // Position linked list: previous pointer
|
||||
[FieldOffset(32)] public EntryState State;
|
||||
|
||||
@@ -66,6 +66,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_spareMemory->PreviousNear = EmptyHandle;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
_spareMemory->Size = (uint)byteSize;
|
||||
}
|
||||
|
||||
@@ -87,6 +89,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_spareMemory->PreviousNear = EmptyHandle;
|
||||
_spareMemory->NextNear = EmptyHandle;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->PrevLock = 0;
|
||||
_spareMemory->NextLock = 0;
|
||||
_spareMemory->Size = (uint)byteSize;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="ISlabAllocatable.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public interface ISlabAllocatable { }
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
# HLSF (Hybrid Layer Segregated Fit) Allocator
|
||||
|
||||
A high-performance, O(1) memory allocator designed for real-time applications that demand predictable allocation and deallocation performance.
|
||||
|
||||
## Overview
|
||||
|
||||
The Hybrid Layer Segregated Fit (HLSF) allocator is a sophisticated memory management system that combines the benefits of segregated free lists with layered size classification. It provides constant-time allocation and deallocation operations while maintaining excellent memory utilization and low fragmentation.
|
||||
|
||||
## Key Features
|
||||
|
||||
### Performance Characteristics
|
||||
- **O(1) Time Complexity**: Both allocation and deallocation operations execute in constant time
|
||||
- **High Throughput**: Capable of handling millions of operations per second
|
||||
- **Low Latency**: Predictable performance suitable for real-time applications
|
||||
- **Excellent Deallocation Speed**: Superior free performance compared to traditional allocators
|
||||
|
||||
### Memory Management
|
||||
- **External Metadata Storage**: All allocation metadata is stored in external SLAB allocators, keeping the main allocation area clean
|
||||
- **Native Byte Alignment Control**: Configurable alignment (minimum 16 bytes) with hardware-optimized defaults
|
||||
- **Large Block Support**: Single allocation limit of 32MB per request
|
||||
- **Efficient Memory Utilization**: Low overhead with intelligent block merging
|
||||
|
||||
### Architecture
|
||||
- **10-Level Hierarchy**: Size classes from 64 bytes to 16MB (64, 256, 1KB, 4KB, 16KB, 64KB, 256KB, 1MB, 4MB, 16MB)
|
||||
- **3-Segment Segregation**: Each level is divided into 3 segments for better size distribution
|
||||
- **Hybrid Approach**: Combines best aspects of segregated fit and layered allocation strategies
|
||||
|
||||
## Architecture Details
|
||||
|
||||
### Size Classification System
|
||||
|
||||
The allocator uses a logarithmic size classification system with 10 levels:
|
||||
|
||||
```
|
||||
Level 0: 64 bytes (2^6)
|
||||
Level 1: 256 bytes (2^8)
|
||||
Level 2: 1KB (2^10)
|
||||
Level 3: 4KB (2^12)
|
||||
Level 4: 16KB (2^14)
|
||||
Level 5: 64KB (2^16)
|
||||
Level 6: 256KB (2^18)
|
||||
Level 7: 1MB (2^20)
|
||||
Level 8: 4MB (2^22)
|
||||
Level 9: 16MB (2^24)
|
||||
```
|
||||
|
||||
Each level follows the pattern: `size = 64 * 4^level`
|
||||
|
||||
### Bucket Organization
|
||||
|
||||
Each size level is further divided into 3 segments based on exact size requirements:
|
||||
- **Segment 0**: 1x level size
|
||||
- **Segment 1**: 2x level size
|
||||
- **Segment 2**: 3x level size
|
||||
|
||||
This provides fine-grained size matching while maintaining O(1) lookup performance.
|
||||
|
||||
### External Metadata Design
|
||||
|
||||
Unlike traditional allocators that embed metadata within allocated blocks, HLSF stores all metadata externally:
|
||||
|
||||
- **Entry Structure (36 bytes)**: Contains position, size, level, and linkage information
|
||||
- **BucketNode Structure (32 bytes)**: Manages free block organization with integrity verification
|
||||
- **SLAB Storage**: Both structures are allocated from separate SLAB allocators for optimal cache performance
|
||||
|
||||
This design provides several advantages:
|
||||
- Clean allocation areas free from metadata corruption
|
||||
- Better cache locality for metadata operations
|
||||
- Simplified debugging and memory inspection
|
||||
- No metadata overhead in allocated blocks
|
||||
|
||||
## Performance Benchmarks
|
||||
|
||||
*Note: The following results are from informal testing and should be considered preliminary.*
|
||||
|
||||
### Single-threaded Performance (65,536 operations)
|
||||
|
||||
| Operation | Average Time | Throughput | Performance Rating |
|
||||
|-----------|--------------|------------|-------------------|
|
||||
| Allocation | 31-40ms | ~2M allocs/sec | Good |
|
||||
| Free | 2-3ms | ~25M frees/sec | Excellent |
|
||||
|
||||
**Test Configuration:**
|
||||
- Pool Size: 16MB
|
||||
- Block Sizes: 64-1024 bytes (mixed)
|
||||
- Platform: x64, .NET 8
|
||||
- Iterations: 5 rounds
|
||||
|
||||
### Memory Efficiency
|
||||
|
||||
- **Metadata Overhead**: ~2.8% of total allocation
|
||||
- **Alignment**: 16-byte default (configurable)
|
||||
- **Fragmentation**: Low due to intelligent merging algorithms
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Allocation
|
||||
|
||||
```csharp
|
||||
// Create allocator with 64MB pool
|
||||
using var allocator = new HybridLayerSegregatedFitAllocator(64 * 1024 * 1024);
|
||||
|
||||
// Allocate 1KB block
|
||||
var handle = allocator.Alloc(1024);
|
||||
|
||||
// Use the allocated memory
|
||||
Span<byte> memory = handle.BufferHandle;
|
||||
memory[0] = 42;
|
||||
|
||||
// Free the block
|
||||
allocator.Free(handle);
|
||||
```
|
||||
|
||||
### Custom Alignment
|
||||
|
||||
```csharp
|
||||
// Create allocator with 32-byte alignment
|
||||
using var allocator = new HybridLayerSegregatedFitAllocator(
|
||||
byteSize: 32 * 1024 * 1024,
|
||||
align: 32
|
||||
);
|
||||
|
||||
var handle = allocator.Alloc(256);
|
||||
// Memory is guaranteed to be 32-byte aligned
|
||||
```
|
||||
|
||||
### Bulk Operations
|
||||
|
||||
```csharp
|
||||
var handles = new List<HybridLayerSegregatedFitAllocationHandle>();
|
||||
|
||||
// Allocate multiple blocks
|
||||
for (int i = 0; i < 1000; i++)
|
||||
{
|
||||
handles.Add(allocator.Alloc((uint)(64 + i % 960))); // 64-1023 bytes
|
||||
}
|
||||
|
||||
// Free all blocks
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
allocator.Free(handle);
|
||||
}
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### HybridLayerSegregatedFitAllocator
|
||||
|
||||
Main allocator class providing memory management functionality.
|
||||
|
||||
#### Constructors
|
||||
|
||||
```csharp
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
```
|
||||
|
||||
#### Methods
|
||||
|
||||
```csharp
|
||||
public HybridLayerSegregatedFitAllocationHandle Alloc(uint size)
|
||||
public void Free(HybridLayerSegregatedFitAllocationHandle handle)
|
||||
public void Dispose()
|
||||
```
|
||||
|
||||
### HybridLayerSegregatedFitAllocationHandle
|
||||
|
||||
Handle representing an allocated memory block.
|
||||
|
||||
#### Properties
|
||||
|
||||
```csharp
|
||||
public readonly byte* Pointer { get; }
|
||||
public readonly Span<byte> BufferHandle { get; }
|
||||
public readonly uint AllocSize { get; }
|
||||
public readonly ref byte this[int index] { get; }
|
||||
```
|
||||
|
||||
## Thread Safety
|
||||
|
||||
The current implementation is **single-threaded**. For multi-threaded applications, external synchronization is required. A concurrent version is planned for future releases.
|
||||
|
||||
## Comparison with Other Allocators
|
||||
|
||||
### vs TLSF (Two-Level Segregated Fit)
|
||||
|
||||
| Aspect | HLSF | TLSF | Notes |
|
||||
|--------|------|------|-------|
|
||||
| Allocation Speed | ~2M ops/sec | ~3-5M ops/sec | HLSF has optimization potential |
|
||||
| Deallocation Speed | ~25M ops/sec | ~10-15M ops/sec | **HLSF advantage** |
|
||||
| Memory Overhead | ~2.8% | ~1-2% | HLSF uses external metadata |
|
||||
| Time Complexity | O(1) | O(1) | Equal |
|
||||
| Code Complexity | Moderate | Low | HLSF more sophisticated |
|
||||
|
||||
### Strengths
|
||||
- Superior deallocation performance
|
||||
- Clean separation of data and metadata
|
||||
- Configurable alignment support
|
||||
- Excellent for allocation-heavy workloads
|
||||
|
||||
### Areas for Improvement
|
||||
- Allocation speed optimization
|
||||
- Memory overhead reduction
|
||||
- Concurrent implementation
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
### Alignment Guarantees
|
||||
|
||||
- Minimum alignment: 16 bytes
|
||||
- All allocations respect specified alignment
|
||||
- Internal blocks aligned to 64-byte boundaries for optimal cache performance
|
||||
|
||||
### Merging Strategy
|
||||
|
||||
The allocator employs an aggressive merging strategy:
|
||||
|
||||
1. **Immediate Merging**: Adjacent free blocks are merged during deallocation
|
||||
2. **Boundary Checking**: Ensures merging respects level boundaries
|
||||
3. **Bidirectional Merging**: Checks both forward and backward neighbors
|
||||
4. **Level Promotion**: Merged blocks may be promoted to higher levels
|
||||
|
||||
## Future Roadmap
|
||||
|
||||
### Planned Optimizations
|
||||
|
||||
1. **Performance Enhancements**
|
||||
- Lookup table optimization for size classification
|
||||
- Bucket operation improvements
|
||||
- SIMD utilization for bulk operations
|
||||
|
||||
2. **Concurrent Support**
|
||||
- Thread-local caches
|
||||
- Lock-free bucket operations
|
||||
- NUMA-aware allocation
|
||||
|
||||
3. **Advanced Features**
|
||||
- Adaptive size level adjustment
|
||||
- Memory pressure handling
|
||||
- Statistics and profiling support
|
||||
|
||||
### Target Performance Goals
|
||||
|
||||
- Allocation Speed: 4-6M ops/sec (single-threaded)
|
||||
- Memory Overhead: <1.5%
|
||||
- Fragmentation Rate: <3%
|
||||
- Concurrent Scaling: Linear up to 16 threads
|
||||
|
||||
## Building and Testing
|
||||
|
||||
The HLSF allocator is part of the StgSharp.Common library and requires:
|
||||
|
||||
- .NET 8 or later
|
||||
- x64 architecture (no x86 support planned)
|
||||
- Windows/Linux
|
||||
|
||||
### Running Benchmarks
|
||||
|
||||
```csharp
|
||||
// See StgSharpDebug/Program.cs for comprehensive benchmark suite
|
||||
var allocator = new HybridLayerSegregatedFitAllocator(16 * 1024 * 1024);
|
||||
// Run allocation/deallocation cycles...
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
This implementation is part of the StgSharp project and follows the project's licensing terms.
|
||||
|
||||
## Contributing
|
||||
|
||||
Performance improvements, bug fixes, and documentation enhancements are welcome. Please ensure all changes maintain the O(1) performance guarantees and include appropriate benchmarks.
|
||||
@@ -0,0 +1,153 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.ChunkedConcurrent.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
{
|
||||
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private ConcurrentBufferStack<nuint> _buffers;
|
||||
private ConcurrentBufferStack<nuint> _recycle;
|
||||
private readonly int _elementSize = Unsafe.SizeOf<T>();
|
||||
|
||||
private nuint _currentBuffer;
|
||||
private nuint _currentCapacity, _currentIndex;
|
||||
|
||||
public ConcurrentChunkedSlabAllocator(nuint count)
|
||||
{
|
||||
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
|
||||
_currentCapacity = count;
|
||||
int initCount = count switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)count
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _currentBuffer + (nuint)(i * _elementSize);
|
||||
}
|
||||
_buffers = new(4);
|
||||
_recycle = BufferStackBuilder.CreateConcurrent(span);
|
||||
_currentCapacity = count;
|
||||
_currentIndex = (nuint)initCount;
|
||||
}
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycle.TryPop(out nuint handle)) {
|
||||
return handle;
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
|
||||
nuint curIndex = Volatile.Read(ref _currentIndex);
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
if (curIndex >= _currentCapacity)
|
||||
{
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
nuint cap = Volatile.Read(ref _currentCapacity);
|
||||
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
|
||||
_buffers.Push(bufferHandle);
|
||||
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
|
||||
_ = Interlocked.Exchange(ref _currentIndex, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
curIndex = Volatile.Read(ref _currentIndex);
|
||||
_ = Interlocked.Add(
|
||||
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
|
||||
return _currentBuffer + ((nuint)_elementSize * curIndex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
NativeMemory.Free((void*)_currentBuffer);
|
||||
while (_buffers.TryPop(out nuint buffer)) {
|
||||
NativeMemory.Free((void*)buffer);
|
||||
}
|
||||
_lock.Dispose();
|
||||
_buffers.Dispose();
|
||||
_recycle.Dispose();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(nuint index)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
return new SlabAllocationHandle<T>(this, false);
|
||||
}
|
||||
|
||||
~ConcurrentChunkedSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.SequencialConcurrent.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
{
|
||||
|
||||
private T* _buffer;
|
||||
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private readonly ConcurrentBufferStack<nuint> _stack;
|
||||
private readonly int _elementSize;
|
||||
private nuint _highestAllocated;
|
||||
private ulong _capacity;
|
||||
|
||||
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
|
||||
{
|
||||
_elementSize = Unsafe.SizeOf<T>();
|
||||
_capacity = initialCapacity;
|
||||
_highestAllocated = 0;
|
||||
|
||||
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
|
||||
int initCount = _capacity switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)_capacity / 4
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = (nuint)i;
|
||||
}
|
||||
_stack = BufferStackBuilder.CreateConcurrent(span);
|
||||
_highestAllocated += (nuint)initCount;
|
||||
}
|
||||
|
||||
internal T* BasePointer
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => _buffer;
|
||||
}
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_stack.TryPop(out nuint index)) {
|
||||
return index;
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
index = Volatile.Read(ref _highestAllocated);
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
if (index >= _capacity)
|
||||
{
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
_capacity = Volatile.Read(ref _capacity) * 2;
|
||||
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
|
||||
return index;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
_stack.Dispose();
|
||||
NativeMemory.Free(_buffer);
|
||||
_lock.Dispose();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading()
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading()
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
|
||||
public override void Free(nuint index)
|
||||
{
|
||||
_stack.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
EnterBufferReading();
|
||||
return new SlabAllocationHandle<T>(this, true);
|
||||
}
|
||||
|
||||
~ConcurrentSequentialSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryHandle.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe struct UnmanagedMemoryHandle
|
||||
{
|
||||
|
||||
public readonly byte* BufferHandle, AllocatorHandle;
|
||||
public readonly nuint Size;
|
||||
|
||||
public UnmanagedMemoryHandle(byte* pointer, byte* handle, nuint s)
|
||||
{
|
||||
BufferHandle = pointer;
|
||||
AllocatorHandle = handle;
|
||||
Size = s;
|
||||
}
|
||||
|
||||
public readonly ref byte this[nuint index]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref Unsafe.AddByteOffset(ref Unsafe.AsRef<byte>(BufferHandle), index);
|
||||
}
|
||||
|
||||
public readonly Enumerator GetEnumerator()
|
||||
{
|
||||
return new Enumerator(this);
|
||||
}
|
||||
|
||||
public ref struct Enumerator : IEnumerator<byte>
|
||||
{
|
||||
|
||||
private nuint _index = nuint.MaxValue;
|
||||
|
||||
private UnmanagedMemoryHandle _handle;
|
||||
|
||||
internal Enumerator(UnmanagedMemoryHandle span)
|
||||
{
|
||||
_handle = span;
|
||||
}
|
||||
|
||||
public ref byte RefCurrent => ref _handle[_index];
|
||||
|
||||
public readonly void Dispose() { }
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
_index++;
|
||||
return _index < _handle.Size;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
_index = 0;
|
||||
}
|
||||
|
||||
object IEnumerator.Current => RefCurrent;
|
||||
|
||||
byte IEnumerator<byte>.Current => RefCurrent;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.BucketTLSF.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
private struct Entry : ISlabAllocatable
|
||||
{
|
||||
|
||||
public Entry* NextNear, PreviousNear;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Drawing;
|
||||
using System.Net.Security;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Transactions;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
private byte* m_Buffer;
|
||||
private SlabAllocator<Entry> _nodes = new(64);
|
||||
|
||||
public
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//-----------------------------------------------------------------------
|
||||
//-----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryHandle.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
//-----------------------------------------------------------------------
|
||||
//-----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Buffers;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public ref struct UnmanagedMemoryHandle
|
||||
{
|
||||
|
||||
public nuint Postion;
|
||||
public Span<byte> Buffer;
|
||||
|
||||
public UnmanagedMemoryHandle() { }
|
||||
|
||||
public ref byte this[int index]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref Buffer[index];
|
||||
}
|
||||
|
||||
public Span<byte>.Enumerator GetEnumerator()
|
||||
{
|
||||
return Buffer.GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.IndexNode.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection.Emit;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
private const int
|
||||
LevelOf4096 = 4,
|
||||
LevelOf1024 = 3,
|
||||
LevelOf256 = 2,
|
||||
LevelOf64 = 1;
|
||||
|
||||
internal readonly ref struct NextIndexNode
|
||||
{
|
||||
|
||||
private readonly Span<M64> _layout;
|
||||
|
||||
public readonly nuint this[int level]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => Position - _layout[level].Member<uint>(0);
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => _layout[level].Member<uint>(0) = (uint)(Position - value);
|
||||
}
|
||||
|
||||
private readonly ref nuint Position
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref _layout[0].Member<nuint>(0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal readonly ref struct PreviousIndexNode
|
||||
{
|
||||
|
||||
private readonly Span<M64> _layout;
|
||||
|
||||
public readonly nuint this[int level]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => Position - _layout[level].Member<uint>(1);
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set => _layout[level].Member<uint>(1) = (uint)(Position - value);
|
||||
}
|
||||
|
||||
private readonly ref nuint Position
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref _layout[0].Member<nuint>(0);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
internal readonly ref struct IndexNode
|
||||
{
|
||||
|
||||
private static readonly uint[] _sizeArr = [16, 64, 256, 1024, 4096];
|
||||
[FieldOffset(0)] private readonly NextIndexNode nxt;
|
||||
[FieldOffset(0)] private readonly PreviousIndexNode pre;
|
||||
[FieldOffset(0)] private readonly Span<M64> _layout;
|
||||
|
||||
public IndexNode(ref byte span)
|
||||
{
|
||||
_layout = MemoryMarshal.CreateSpan(ref Unsafe.As<byte, M64>(ref span), 8);
|
||||
}
|
||||
|
||||
public readonly ref nuint Position => ref _layout[0].Member<nuint>(0);
|
||||
|
||||
public readonly ref uint Size => ref _layout[5].Member<uint>(0);
|
||||
|
||||
public readonly ref int Level => ref _layout[5].Member<int>(1);
|
||||
|
||||
public readonly ref nuint Segment => ref _layout[6].Member<nuint>(0);
|
||||
|
||||
public readonly uint BlockSize => LevelToSize(Level);
|
||||
|
||||
public static uint LevelToSize(int level)
|
||||
{
|
||||
return level switch
|
||||
{
|
||||
< 0 or > 4 => throw new ArgumentOutOfRangeException(
|
||||
nameof(level), "Level must be from 1 to 4."),
|
||||
_ => _sizeArr[level]
|
||||
};
|
||||
}
|
||||
|
||||
#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
|
||||
public readonly ref readonly NextIndexNode Next
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref nxt;
|
||||
}
|
||||
|
||||
public readonly ref readonly PreviousIndexNode Previous
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => ref pre;
|
||||
}
|
||||
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.IndexOperation.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void ConnectToNodeAtLevel(
|
||||
IndexNode previous,
|
||||
IndexNode current,
|
||||
int level,
|
||||
bool isHead)
|
||||
{
|
||||
if (isHead)
|
||||
{
|
||||
_entrance[level] = current.Position;
|
||||
current.Previous[level] = 0;
|
||||
} else
|
||||
{
|
||||
previous.Next[level] = current.Previous[level] = (uint)(current.Position - previous.Position);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private IndexNode GetNode(nuint pos)
|
||||
{
|
||||
ref byte span = ref *(_buffer + pos);
|
||||
IndexNode node = new(ref span)
|
||||
{
|
||||
Position = pos
|
||||
};
|
||||
return node;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsNodeBeginning(nuint pos, int level)
|
||||
{
|
||||
return _entrance[level] == pos;
|
||||
}
|
||||
|
||||
private bool Rebase(IndexNode n, uint offset)
|
||||
{
|
||||
nuint pos = n.Position;
|
||||
uint size = n.Size;
|
||||
if (size <= offset + 64)
|
||||
{
|
||||
for (int i = 0; i <= n.Level; i++) {
|
||||
RemoveLevelHead(n, i);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
uint
|
||||
newOffset = ((offset / 64) + 1) * 64,
|
||||
remain = n.Size - newOffset,
|
||||
c64 = remain % 4,
|
||||
c256 = remain / 4,
|
||||
c1024 = c256 / 4,
|
||||
c4096 = c1024 / 4;
|
||||
c256 %= 4;
|
||||
c1024 %= 4;
|
||||
IndexNode
|
||||
/*
|
||||
* level is not important here, higher level will get 0 Size,
|
||||
* and ignored in coming logic
|
||||
*/
|
||||
p64 = GetNode(n.Position - n.Previous[1]),
|
||||
p256 = GetNode(n.Position - n.Previous[2]),
|
||||
p1024 = GetNode(n.Position - n.Previous[3]),
|
||||
p4096 = GetNode(n.Position - n.Previous[4]),
|
||||
n64 = GetNode(n.Position + n.Previous[1]),
|
||||
n256 = GetNode(n.Position + n.Previous[2]),
|
||||
n1024 = GetNode(n.Position + n.Previous[3]),
|
||||
n4096 = GetNode(n.Position + n.Previous[4]);
|
||||
IndexNode nTemp;
|
||||
if (c64 != 0)
|
||||
{
|
||||
// set param to separated 64 bytes block and set next and previous offset to it.
|
||||
size = c64 * 64;
|
||||
pos += size;
|
||||
nTemp = GetNode(pos);
|
||||
nTemp.Position = pos;
|
||||
nTemp.Size = size;
|
||||
ConnectToNodeAtLevel(p64, nTemp, 1, n.Position == _entrance[1]);
|
||||
p64 = nTemp;
|
||||
}
|
||||
if (c256 != 0)
|
||||
{
|
||||
size = c256 * 256;
|
||||
pos += size;
|
||||
nTemp = GetNode(pos);
|
||||
nTemp.Position = pos;
|
||||
nTemp.Size = size;
|
||||
nTemp.Previous[2] = n.Previous[2] + size;
|
||||
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
|
||||
ConnectToNodeAtLevel(p256, nTemp, 2, n.Position == _entrance[2]);
|
||||
p64 = p256 = nTemp;
|
||||
}
|
||||
if (c1024 != 0)
|
||||
{
|
||||
size = c1024 * 1024;
|
||||
pos += size;
|
||||
nTemp = GetNode(pos);
|
||||
nTemp.Position = pos;
|
||||
nTemp.Size = size;
|
||||
nTemp.Previous[2] = n.Previous[2] + size;
|
||||
|
||||
// connect to previous node
|
||||
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
|
||||
ConnectToNodeAtLevel(p256, nTemp, 2, p256.Position == _entrance[2]);
|
||||
ConnectToNodeAtLevel(p1024, nTemp, 3, n.Position == _entrance[3]);
|
||||
p64 = p256 = p1024 = nTemp;
|
||||
}
|
||||
if (c4096 != 0)
|
||||
{
|
||||
size = c1024 * 1024;
|
||||
pos += size;
|
||||
nTemp = GetNode(pos);
|
||||
nTemp.Position = pos;
|
||||
nTemp.Size = size;
|
||||
nTemp.Previous[2] = n.Previous[2] + size;
|
||||
|
||||
// connect to previous node
|
||||
ConnectToNodeAtLevel(p64, nTemp, 1, p64.Position == _entrance[1]);
|
||||
ConnectToNodeAtLevel(p256, nTemp, 2, p256.Position == _entrance[2]);
|
||||
ConnectToNodeAtLevel(p1024, nTemp, 3, p1024.Position == _entrance[3]);
|
||||
ConnectToNodeAtLevel(p4096, nTemp, 4, n.Position == _entrance[4]);
|
||||
p64 = p256 = p1024 = p4096 = nTemp;
|
||||
}
|
||||
|
||||
ConnectToNodeAtLevel(p64, n64, 1, false);
|
||||
ConnectToNodeAtLevel(
|
||||
p256, n256, 2, _entrance[2] == n.Position && n256.Position == n.Position);
|
||||
ConnectToNodeAtLevel(
|
||||
p1024, n1024, 3, _entrance[3] == n.Position && n1024.Position == n.Position);
|
||||
ConnectToNodeAtLevel(
|
||||
p4096, n4096, 4, _entrance[4] == n.Position && n4096.Position == n.Position);
|
||||
return true;
|
||||
}
|
||||
|
||||
private void RemoveLevelHead(IndexNode n, int level)
|
||||
{
|
||||
IndexNode nxt, pre;
|
||||
if (n.Previous[level] == 0)
|
||||
{
|
||||
// head of usable LEVEL;
|
||||
nxt = GetNode(n.Position + n.Next[level]);
|
||||
_entrance[level] = nxt.Position;
|
||||
}
|
||||
pre = GetNode(n.Previous[level]);
|
||||
nxt = GetNode(n.Next[level]);
|
||||
nxt.Previous[level] = nxt.Previous[level] + n.Previous[level];
|
||||
pre.Next[level] = pre.Next[level] + n.Next[level];
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.MegaManage.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
private readonly LinkedList<MegaBlockHead> _megaAlloc = new();
|
||||
|
||||
private readonly LinkedList<MegaBlockHead> _megaEmpty = new();
|
||||
private MegaBlockHead _megaFirst;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void InsertMegaNear(MegaBlockHead previous, MegaBlockHead current)
|
||||
{
|
||||
MegaBlockHead pp = current.PreviousNear;
|
||||
pp.NextNear = previous;
|
||||
previous.PreviousNear = pp;
|
||||
previous.NextNear = current;
|
||||
current.PreviousNear = previous;
|
||||
}
|
||||
|
||||
private static void InsertMegaSorted(LinkedList<MegaBlockHead> list, MegaBlockHead newItem)
|
||||
{
|
||||
LinkedListNode<MegaBlockHead>? node = list.First;
|
||||
|
||||
while (node != null && node.Value.Position < newItem.Position) {
|
||||
node = node.Next;
|
||||
}
|
||||
|
||||
_ = node != null ? list.AddBefore(node, newItem) : list.AddLast(newItem);
|
||||
}
|
||||
|
||||
private bool TryGetMegaBlock(nuint size, out nuint position, int align = 1)
|
||||
{
|
||||
LinkedListNode<MegaBlockHead> node = _megaEmpty.First!;
|
||||
while (node is not null)
|
||||
{
|
||||
MegaBlockHead block = node.Value;
|
||||
if (block.Size >= size + (nuint)align)
|
||||
{
|
||||
position = block.Position;
|
||||
nuint s = position + (nuint)(align - 1);
|
||||
MegaBlockHead b;
|
||||
if (block.Rebase(s))
|
||||
{
|
||||
b = new()
|
||||
{
|
||||
Position = position,
|
||||
Size = position + (nuint)(align - 1),
|
||||
Usage = MegaBlockUsage.MegaAlloc,
|
||||
};
|
||||
InsertMegaSorted(_megaAlloc, b);
|
||||
InsertMegaNear(b, block);
|
||||
} else
|
||||
{
|
||||
b = new()
|
||||
{
|
||||
Position = position,
|
||||
Size = block.Size,
|
||||
Usage = MegaBlockUsage.MegaAlloc,
|
||||
};
|
||||
InsertMegaSorted(_megaAlloc, b);
|
||||
_megaEmpty.Remove(node);
|
||||
InsertMegaNear(b, block.NextNear);
|
||||
}
|
||||
position = s & (nuint)~(align - 1);
|
||||
return true;
|
||||
}
|
||||
node = node.Next!;
|
||||
}
|
||||
position = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
private unsafe class MegaBlockHead
|
||||
{
|
||||
|
||||
private readonly ulong[] Heads = new ulong[4];
|
||||
|
||||
public MegaBlockHead NextNear { get; set; }
|
||||
|
||||
public MegaBlockHead PreviousNear { get; set; }
|
||||
|
||||
public MegaBlockUsage Usage { get; set; }
|
||||
|
||||
public nuint Position { get; set; }
|
||||
|
||||
public nuint Size { get; set; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ref nuint DagHead(int level)
|
||||
{
|
||||
return ref Unsafe.As<ulong, nuint>(ref Heads[level]);
|
||||
}
|
||||
|
||||
public bool Rebase(nuint size)
|
||||
{
|
||||
nuint s = this.Size - size;
|
||||
if (s > 1024 * 1024 * 128)//space larger than 128MB is considered a new mega block
|
||||
{
|
||||
Position += size;
|
||||
this.Size -= size;
|
||||
Usage = MegaBlockUsage.EmptyDag;
|
||||
return true;
|
||||
}
|
||||
MegaBlockHead pre = PreviousNear, nxt = NextNear;
|
||||
pre.NextNear = nxt;
|
||||
nxt.PreviousNear = pre;
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private enum MegaBlockUsage
|
||||
{
|
||||
|
||||
EmptyDag = 0,
|
||||
UsedDag = 1,
|
||||
MegaAlloc = 2
|
||||
|
||||
}
|
||||
|
||||
private MegaBlockHead
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="UnmanagedMemoryPool.cs"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.Security;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
using System.Transactions;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public unsafe partial class UnmanagedMemoryPool
|
||||
{
|
||||
|
||||
private readonly byte* _buffer;
|
||||
|
||||
// 4k,1k,256b,64b,16b
|
||||
private nuint[] _entrance = new nuint[5];
|
||||
private bool _disposedValue;
|
||||
private readonly nuint _initialCapacity;
|
||||
|
||||
internal UnmanagedMemoryPool(nuint initialCapacity, int memoryAlign)
|
||||
{
|
||||
_initialCapacity = initialCapacity;
|
||||
_buffer = (byte*)NativeMemory.AlignedAlloc(initialCapacity, (nuint)memoryAlign);
|
||||
}
|
||||
|
||||
public UnmanagedMemoryHandle Allocate(uint size = 0)
|
||||
{
|
||||
if (size == 0) {
|
||||
return new UnmanagedMemoryHandle
|
||||
{
|
||||
Postion = 0,
|
||||
Buffer = new Span<byte>(_buffer, 0)
|
||||
};
|
||||
}
|
||||
if (size > 0x8000_0000) {
|
||||
throw new ArgumentOutOfRangeException(
|
||||
"Size of buffer cannot be greater than 2147483648");
|
||||
}
|
||||
int level = GetLevelFromSize(size);
|
||||
IndexNode allocator = GetNode(_entrance[level]);
|
||||
nuint previous = allocator.Previous[level];
|
||||
if (allocator.Size < size)
|
||||
{
|
||||
previous = allocator.Position;
|
||||
allocator = GetNode(allocator.Next[level]);
|
||||
}
|
||||
bool isOverflow = false;
|
||||
checked
|
||||
{
|
||||
try
|
||||
{
|
||||
isOverflow = (uint)(allocator.Position - previous + size) > 0xC0000000;
|
||||
}
|
||||
catch (OverflowException)
|
||||
{
|
||||
isOverflow = true;
|
||||
}
|
||||
}
|
||||
if (isOverflow)
|
||||
{
|
||||
// offset is too large
|
||||
}
|
||||
|
||||
UnmanagedMemoryHandle handle = new()
|
||||
{
|
||||
Postion = allocator.Position,
|
||||
Buffer = new Span<byte>(_buffer + allocator.Position, (int)size),
|
||||
};
|
||||
Rebase(allocator, size);
|
||||
return handle;
|
||||
}
|
||||
|
||||
public static UnmanagedMemoryPool Create(nuint initialCapacity, int memoryAlign = 16)
|
||||
{
|
||||
UnmanagedMemoryPool pool = new(initialCapacity, memoryAlign);
|
||||
IndexNode node = pool.GetNode(0);
|
||||
for (int i = 4; i > 0; i--)
|
||||
{
|
||||
node.Previous[i] = 0;
|
||||
node.Next[i] = IndexNode.LevelToSize(i);
|
||||
}
|
||||
node.Size = (uint)initialCapacity;
|
||||
pool._megaFirst = new MegaBlockHead
|
||||
{
|
||||
Position = 0,
|
||||
Size = initialCapacity,
|
||||
Usage = MegaBlockUsage.EmptyDag
|
||||
};
|
||||
_ = pool._megaEmpty.AddLast(pool._megaFirst);
|
||||
return pool;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public void Free(UnmanagedMemoryHandle handle)
|
||||
{
|
||||
if (_buffer + handle.Postion != Unsafe.AsPointer(ref handle.Buffer[0])) {
|
||||
throw new InvalidOperationException("Handle does not belong to pool");
|
||||
}
|
||||
nuint location = handle.Postion;
|
||||
IndexNode root = GetNode(_entrance[4]);
|
||||
while (root.Position > location) { }
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!_disposedValue)
|
||||
{
|
||||
if (disposing) { }
|
||||
|
||||
NativeMemory.AlignedFree(_buffer);
|
||||
_entrance = null!;
|
||||
_disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetLevelFromSize(uint size)
|
||||
{
|
||||
int level = 0;
|
||||
uint temp = size / 8;
|
||||
while (temp > 3)
|
||||
{
|
||||
level++;
|
||||
temp >>= 2;
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
~UnmanagedMemoryPool()
|
||||
{
|
||||
Dispose(disposing:false);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user