HLSf use nuint rather than long, L4 now uses hlsf rather than array to alloc cache line.
This commit is contained in:
@@ -31,7 +31,7 @@ using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Common.Collections
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public class ConcurrentFlexibleArray<T>(
|
||||
int capacity = 4
|
||||
|
||||
@@ -32,7 +32,7 @@ using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Common.Collections
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Zero-overhead view over a contiguous region of unmanaged memory. Does not own the memory —
|
||||
@@ -125,7 +125,7 @@ namespace StgSharp.Common.Collections
|
||||
int length
|
||||
)
|
||||
{
|
||||
Debug.Assert((uint)start + (uint)length <= (uint)Length,
|
||||
Debug.Assert(((uint)start) + ((uint)length) <= (uint)Length,
|
||||
"UnmanagedArray.AsSpan range out of bounds");
|
||||
return new Span<T>(Address + start, length);
|
||||
}
|
||||
@@ -144,7 +144,7 @@ namespace StgSharp.Common.Collections
|
||||
T* ptr
|
||||
)
|
||||
{
|
||||
int offset = (int)(((nint)ptr - (nint)Address) / ElementSize);
|
||||
int offset = (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(offset, nameof(ptr));
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, Length, nameof(ptr));
|
||||
return offset;
|
||||
@@ -169,7 +169,7 @@ namespace StgSharp.Common.Collections
|
||||
T* ptr
|
||||
)
|
||||
{
|
||||
return (int)(((nint)ptr - (nint)Address) / ElementSize);
|
||||
return (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.Mathematics;
|
||||
using StgSharp.HighPerformance;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="Global"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -31,7 +31,8 @@ global using System;
|
||||
global using System.Runtime.CompilerServices;
|
||||
global using System.Threading;
|
||||
|
||||
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.HighPerformance")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.RegularAnalysis")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.Graphics")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
|
||||
[assembly: InternalsVisibleTo("StgSharpDebug")]
|
||||
|
||||
@@ -29,7 +29,7 @@ using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
|
||||
public unsafe class BufferStack<T> : IDisposable where T: unmanaged
|
||||
|
||||
@@ -33,7 +33,7 @@ using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public static class BufferStackBuilder
|
||||
{
|
||||
@@ -104,11 +104,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
AllocateBuffer(capacity);
|
||||
}
|
||||
|
||||
public ulong Count
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => (ulong)Volatile.Read(ref _count);
|
||||
}
|
||||
public ulong Count => (ulong)Volatile.Read(ref _count);
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
@@ -191,7 +187,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
if (header->Top >= header->Capacity)
|
||||
{
|
||||
int newCap = header->Capacity * 2;
|
||||
nuint newSize = (nuint)sizeof(Header) + ((nuint)newCap * (nuint)sizeof(T));
|
||||
nuint newSize = ((nuint)sizeof(Header)) + (((nuint)newCap) * ((nuint)sizeof(T)));
|
||||
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
@@ -291,8 +287,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
|
||||
fixed (T* sourcePtr = span) {
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart, (nuint)header->Capacity * (nuint)sizeof(T),
|
||||
(nuint)span.Length * (nuint)sizeof(T));
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart,
|
||||
((nuint)header->Capacity) * ((nuint)sizeof(T)),
|
||||
((nuint)span.Length) * ((nuint)sizeof(T)));
|
||||
}
|
||||
|
||||
header->Top = span.Length;
|
||||
@@ -303,7 +300,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
nuint size = (nuint)sizeof(Header) + (nuint)capacity * (nuint)sizeof(T);
|
||||
nuint size = ((nuint)sizeof(Header)) + (((nuint)capacity) * ((nuint)sizeof(T)));
|
||||
_buffer = (byte*)NativeMemory.Alloc(size);
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
|
||||
@@ -1,30 +1,27 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocationHandle.cs"
|
||||
// file="HLSFAllocationHandle"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -35,17 +32,20 @@ using System.Drawing;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
using Allocator = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
|
||||
using Allocator = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public readonly unsafe struct hlsfHandle
|
||||
public readonly unsafe struct HlsfHandle
|
||||
{
|
||||
|
||||
internal readonly Allocator.Entry* EntryHandle;
|
||||
public readonly long AllocSize;
|
||||
public readonly nuint AllocSize;
|
||||
|
||||
internal hlsfHandle(Allocator.Entry* handle, long s)
|
||||
internal HlsfHandle(
|
||||
Allocator.Entry* handle,
|
||||
nuint s
|
||||
)
|
||||
{
|
||||
EntryHandle = handle;
|
||||
Pointer = (byte*)handle->Position;
|
||||
@@ -54,7 +54,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
|
||||
}
|
||||
|
||||
public readonly ref byte this[uint index]
|
||||
public readonly ref byte this[
|
||||
uint index
|
||||
]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get
|
||||
@@ -66,11 +68,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
public readonly byte* Pointer { get; init; }
|
||||
|
||||
public readonly Span<byte> BufferHandle
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => new(Pointer, (int)AllocSize);
|
||||
}
|
||||
public readonly Span<byte> BufferHandle => new(Pointer, (int)AllocSize);
|
||||
|
||||
public readonly Enumerator GetEnumerator()
|
||||
{
|
||||
@@ -80,11 +78,13 @@ namespace StgSharp.Mathematics.Memory
|
||||
public ref struct Enumerator : IEnumerator<byte>
|
||||
{
|
||||
|
||||
private hlsfHandle _handle;
|
||||
private HlsfHandle _handle;
|
||||
|
||||
private uint _index = ~0u;
|
||||
|
||||
internal Enumerator(hlsfHandle span)
|
||||
internal Enumerator(
|
||||
HlsfHandle span
|
||||
)
|
||||
{
|
||||
_handle = span;
|
||||
Span = span.BufferHandle;
|
||||
|
||||
@@ -1,43 +1,43 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Alloc.cs"
|
||||
// file="HLSFAllocator.Alloc"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
|
||||
using System.Reflection.Metadata;
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
public partial hlsfHandle Alloc(long size)
|
||||
public partial HlsfHandle Alloc(
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
if (size > _maxSize) {
|
||||
throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
|
||||
@@ -70,18 +70,72 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
if (handle != null)
|
||||
{
|
||||
int requestedOffset = (segmentIndex + 1) * _levelSizeArray[level];
|
||||
if (requestedOffset > handle->Size) {
|
||||
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
|
||||
if (requestedOffset > (int)handle->Size) {
|
||||
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
|
||||
}
|
||||
long remainder = handle->Size - requestedOffset;
|
||||
nuint remainder = handle->Size - ((nuint)requestedOffset);
|
||||
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
|
||||
handle->State = EntryState.Alloc;
|
||||
if (remainder > 0) {
|
||||
SliceMemory(handle->NextNear, remainder);
|
||||
}
|
||||
}
|
||||
return new hlsfHandle(handle, size);
|
||||
return new(handle, size);
|
||||
}
|
||||
|
||||
public bool TryAlloc(
|
||||
nuint size,
|
||||
out HlsfHandle handle
|
||||
)
|
||||
{
|
||||
if (size > _maxSize)
|
||||
{
|
||||
handle = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
int level = GetLevelFromSize(size);
|
||||
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
|
||||
|
||||
int i = level, s = segmentIndex;
|
||||
Entry* entry;
|
||||
while (!TryPopLevelForAllocation(i, s, out entry))
|
||||
{
|
||||
if (++s >= 3)
|
||||
{
|
||||
s = 0;
|
||||
i++;
|
||||
}
|
||||
if (i > MaxLevel)
|
||||
{
|
||||
if (TryPopLevelForAllocation(10, 1, out entry))
|
||||
{
|
||||
break;
|
||||
} else if (TryAllocateNew32MBBlock(out entry))
|
||||
{
|
||||
break;
|
||||
}
|
||||
handle = default;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (entry != null)
|
||||
{
|
||||
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
|
||||
if (requestedOffset > (int)entry->Size) {
|
||||
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
|
||||
}
|
||||
nuint remainder = entry->Size - ((nuint)requestedOffset);
|
||||
SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
|
||||
entry->State = EntryState.Alloc;
|
||||
if (remainder > 0) {
|
||||
SliceMemory(entry->NextNear, remainder);
|
||||
}
|
||||
}
|
||||
handle = new(entry, size);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -94,28 +148,31 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// <param name="sizeToSlice">
|
||||
///
|
||||
/// </param>
|
||||
private void SliceMemory(Entry* e, long sizeToSlice)
|
||||
private void SliceMemory(
|
||||
Entry* e,
|
||||
nuint sizeToSlice
|
||||
)
|
||||
{
|
||||
Entry* next = e;
|
||||
long remain = sizeToSlice; // remained size to slice
|
||||
long offset = (long)e->Position - sizeToSlice - (long)m_Buffer; // offset of buffer to slice
|
||||
long end = (long)e->Position - (long)m_Buffer; // end offset of buffer to slice
|
||||
nuint remain = sizeToSlice; // remained size to slice
|
||||
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
|
||||
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
|
||||
|
||||
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
|
||||
i = int.Min(10, i);
|
||||
for (; i < _levelSizeArray.Length; i++)
|
||||
{
|
||||
int size = _levelSizeArray[i];
|
||||
int upper = size * 4;
|
||||
long edge = (offset + upper - 1) / upper * upper;
|
||||
int count = (int)((long.Min(edge, end) - offset) / size);
|
||||
uint size = _levelSizeArray[i];
|
||||
nuint upper = size * 4;
|
||||
nuint edge = (((offset + upper) - 1U) / upper) * upper;
|
||||
int count = (int)((nuint.Min(edge, end) - offset) / size);
|
||||
if (count > 0)
|
||||
{
|
||||
long newSize = count * size;
|
||||
nuint newSize = ((nuint)count) * size;
|
||||
|
||||
Entry* newEntry = (Entry*)_entries.Allocate();
|
||||
InsertEntryBefore(newEntry, next);
|
||||
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
|
||||
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
|
||||
newEntry->State = EntryState.Empty;
|
||||
|
||||
BucketNode* b = (BucketNode*)newEntry->Position;
|
||||
@@ -131,17 +188,17 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
if (i > MaxLevel)
|
||||
{
|
||||
int size = _levelSizeArray[^1];
|
||||
int upper = size * 4;
|
||||
long edge = end / upper * upper;
|
||||
int count = (int)((long.Min(edge, end) - offset) / upper);
|
||||
uint size = _levelSizeArray[^1];
|
||||
nuint upper = size * 4;
|
||||
nuint edge = (end / upper) * upper;
|
||||
int count = (int)((nuint.Min(edge, end) - offset) / upper);
|
||||
if (count > 0)
|
||||
{
|
||||
long newSize = (long)count * (long)upper;
|
||||
nuint newSize = ((nuint)count) * upper;
|
||||
|
||||
Entry* newEntry = (Entry*)_entries.Allocate();
|
||||
InsertEntryBefore(newEntry, next);
|
||||
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
|
||||
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
|
||||
newEntry->State = EntryState.Empty;
|
||||
|
||||
BucketNode* b = (BucketNode*)newEntry->Position;
|
||||
@@ -157,16 +214,16 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
for (i--; i >= 0; i--)
|
||||
{
|
||||
int size = _levelSizeArray[i];
|
||||
long edge = end / size * size;
|
||||
int count = (int)((long.Min(edge, end) - offset) / size);
|
||||
uint size = _levelSizeArray[i];
|
||||
nuint edge = (end / size) * size;
|
||||
int count = (int)((nuint.Min(edge, end) - offset) / size);
|
||||
if (count > 0)
|
||||
{
|
||||
long newSize = count * size;
|
||||
nuint newSize = ((nuint)count) * size;
|
||||
|
||||
Entry* newEntry = (Entry*)_entries.Allocate();
|
||||
InsertEntryBefore(newEntry, next);
|
||||
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
|
||||
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
|
||||
newEntry->State = EntryState.Empty;
|
||||
|
||||
BucketNode* b = (BucketNode*)newEntry->Position;
|
||||
@@ -192,16 +249,18 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// <returns>
|
||||
/// True if allocation succeeded
|
||||
/// </returns>
|
||||
private bool TryAllocateNew32MBBlock(out Entry* handle)
|
||||
private bool TryAllocateNew32MBBlock(
|
||||
out Entry* handle
|
||||
)
|
||||
{
|
||||
nuint pos = _spareMemory->Position;
|
||||
long remain = _spareMemory->Size;
|
||||
nuint remain = _spareMemory->Size;
|
||||
if (remain <= 0)
|
||||
{
|
||||
handle = null;
|
||||
return false;
|
||||
}
|
||||
long size = remain > 32 * 1024 * 1024L ? 32 * 1024 * 1024L : remain;
|
||||
nuint size = (remain > 32 * 1024 * 1024L) ? (32 * 1024 * 1024U) : remain;
|
||||
|
||||
|
||||
// build new entry
|
||||
@@ -212,7 +271,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
// set fields of spare memory
|
||||
_spareMemory->Size -= size;
|
||||
_spareMemory->Position += (nuint)size;
|
||||
_spareMemory->Position += size;
|
||||
|
||||
|
||||
handle = e;
|
||||
@@ -222,7 +281,11 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// <summary>
|
||||
/// Helper method for allocation - converts BucketNode to Entry
|
||||
/// </summary>
|
||||
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
|
||||
private bool TryPopLevelForAllocation(
|
||||
int levelIndex,
|
||||
int sizeIndex,
|
||||
out Entry* entry
|
||||
)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
|
||||
{
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.BucketNode"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -30,18 +30,22 @@ using System.Reflection.Metadata;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private void PushBucketToLevel(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
private void PushBucketToLevel(
|
||||
int levelIndex,
|
||||
int sizeIndex,
|
||||
BucketNode* node
|
||||
)
|
||||
{
|
||||
if (node == null || node->EntryRef == null) {
|
||||
if ((node == null) || (node->EntryRef == null)) {
|
||||
return;
|
||||
}
|
||||
|
||||
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
|
||||
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
|
||||
if (_bucketHeads[index] is null)
|
||||
{
|
||||
_bucketHeads[index] = node;
|
||||
@@ -69,9 +73,13 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
private void RemoveFromBucket(
|
||||
int levelIndex,
|
||||
int sizeIndex,
|
||||
BucketNode* node
|
||||
)
|
||||
{
|
||||
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
|
||||
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
|
||||
if (_bucketHeads[index] == node)
|
||||
{
|
||||
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
|
||||
@@ -94,9 +102,13 @@ namespace StgSharp.Mathematics.Memory
|
||||
return;
|
||||
}
|
||||
|
||||
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
|
||||
private bool TryPopLevel(
|
||||
int levelIndex,
|
||||
int sizeIndex,
|
||||
out BucketNode* node
|
||||
)
|
||||
{
|
||||
Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex;
|
||||
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
|
||||
|
||||
BucketNode* head = _bucketHeads[index];
|
||||
|
||||
@@ -125,9 +137,13 @@ namespace StgSharp.Mathematics.Memory
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
|
||||
private bool TryRemoveFromBucket(
|
||||
int levelIndex,
|
||||
int sizeIndex,
|
||||
BucketNode* node
|
||||
)
|
||||
{
|
||||
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
|
||||
Index index = (levelIndex > 9) ? (^1) : ((3 * levelIndex) + sizeIndex);
|
||||
if (_bucketHeads[index] == node)
|
||||
{
|
||||
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
|
||||
|
||||
@@ -31,7 +31,7 @@ using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed class HLSFPositionChainBreakException : Exception
|
||||
{
|
||||
|
||||
@@ -1,30 +1,27 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Free.cs"
|
||||
// file="HLSFAllocator.Free"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -34,13 +31,14 @@ using System.Collections.Generic;
|
||||
using System.Drawing;
|
||||
using System.Linq;
|
||||
using System.Reflection.Emit;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public enum FreePolicy
|
||||
{
|
||||
@@ -61,8 +59,8 @@ namespace StgSharp.Mathematics.Memory
|
||||
public partial void Collect()
|
||||
{
|
||||
Entry* cur = _spareMemory->NextNear;
|
||||
long size = 0;
|
||||
while (cur != null && cur != _spareMemory)
|
||||
nuint size = 0;
|
||||
while ((cur != null) && (cur != _spareMemory))
|
||||
{
|
||||
// cur is not empty, skip
|
||||
if (cur->State != EntryState.Empty)
|
||||
@@ -79,7 +77,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
|
||||
RemoveFromPositionChain(cur);
|
||||
cur = next;
|
||||
} while (cur != null && cur != _spareMemory && cur->State == EntryState.Empty);
|
||||
} while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
|
||||
if (size > 0) {
|
||||
SliceMemory(cur, size);
|
||||
}
|
||||
@@ -87,13 +85,16 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// <param name="collectPolicy">
|
||||
/// </param>
|
||||
public void Free(hlsfHandle handle, FreePolicy collectPolicy = FreePolicy.NoCollect)
|
||||
/// Free allocated memory block and perform adjacent block merging
|
||||
/// </summary>
|
||||
/// <param name="handle">
|
||||
/// Handle to the allocated memory block
|
||||
/// <param name="collectPolicy">
|
||||
/// </param>
|
||||
public void Free(
|
||||
HlsfHandle handle,
|
||||
FreePolicy collectPolicy = FreePolicy.NoCollect
|
||||
)
|
||||
{
|
||||
Entry* e = handle.EntryHandle;
|
||||
if (e == null) {
|
||||
@@ -113,7 +114,44 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
// Calculate final level and segment for the merged block
|
||||
long finalSize = e->Size;
|
||||
nuint finalSize = e->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
e->Level = finalLevel;
|
||||
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
|
||||
|
||||
// Push the merged entry back to appropriate bucket
|
||||
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
|
||||
|
||||
// Console.WriteLine("one block freed");
|
||||
if (collectPolicy == FreePolicy.FullCollect)
|
||||
{
|
||||
Collect();
|
||||
}
|
||||
}
|
||||
|
||||
internal void Free(
|
||||
Entry* e,
|
||||
FreePolicy collectPolicy = FreePolicy.NoCollect
|
||||
)
|
||||
{
|
||||
if (e == null) {
|
||||
return;
|
||||
}
|
||||
if (e->State != EntryState.Alloc) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Mark as empty so it can be merged
|
||||
e->State = EntryState.Empty;
|
||||
|
||||
// Try merge with adjacent free blocks and remove them from buckets
|
||||
if (collectPolicy == FreePolicy.LazyCollect)
|
||||
{
|
||||
MergeAndRemoveFromBuckets(e);
|
||||
}
|
||||
|
||||
// Calculate final level and segment for the merged block
|
||||
nuint finalSize = e->Size;
|
||||
int finalLevel = GetLevelFromSize(finalSize);
|
||||
e->Level = finalLevel;
|
||||
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.Level"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -33,23 +33,38 @@ using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
|
||||
private const int MaxLevel = 9;
|
||||
|
||||
private int[] _levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216];
|
||||
private uint[] _levelSizeArray = [
|
||||
64U,
|
||||
256U,
|
||||
1024U,
|
||||
4096U,
|
||||
16384U,
|
||||
65536U,
|
||||
262144U,
|
||||
1048576U,
|
||||
4194304U,
|
||||
16777216U
|
||||
];
|
||||
|
||||
public int AlignOfLevel(int bucketLevel)
|
||||
public int AlignOfLevel(
|
||||
int bucketLevel
|
||||
)
|
||||
{
|
||||
int blockSize = _levelSizeArray[bucketLevel];
|
||||
return int.Min(blockSize, _align);
|
||||
uint blockSize = _levelSizeArray[bucketLevel];
|
||||
return Math.Min((int)blockSize, _align);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetLevelFromSize(long size)
|
||||
private static int GetLevelFromSize(
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
/*
|
||||
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
|
||||
@@ -72,7 +87,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int GetLevelFromSize(uint size)
|
||||
private static int GetLevelFromSize(
|
||||
uint size
|
||||
)
|
||||
{
|
||||
return GetLevelFromSize(size);
|
||||
}
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.SegmentEntry"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -29,7 +29,7 @@ using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator
|
||||
{
|
||||
@@ -38,37 +38,51 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// Check if two Entries are adjacent in memory
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool AssertAdjacentTo(Entry* current, Entry* other)
|
||||
private static bool AssertAdjacentTo(
|
||||
Entry* current,
|
||||
Entry* other
|
||||
)
|
||||
{
|
||||
// Only a predicate; never throw here. Callers rely on false to skip merging.
|
||||
return other->Position + (nuint)other->Size == current->Position ||
|
||||
current->Position + (nuint)current->Size == other->Position;
|
||||
return (other->Position + other->Size == current->Position) ||
|
||||
(current->Position + current->Size == other->Position);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// check out if a bucket is within the same higher-level boundary as currentOffset
|
||||
/// </summary>
|
||||
private bool AssertBoundary(Entry* previous, Entry* current, int level)
|
||||
private bool AssertBoundary(
|
||||
Entry* previous,
|
||||
Entry* current,
|
||||
int level
|
||||
)
|
||||
{
|
||||
level = int.Min(level, MaxLevel);
|
||||
int nextLevelSize = _levelSizeArray[level] * 4;
|
||||
ulong prevHead = previous->Position - (ulong)m_Buffer,
|
||||
currHead = current->Position - (ulong)m_Buffer;
|
||||
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
|
||||
uint nextLevelSize = _levelSizeArray[level] * 4;
|
||||
ulong prevHead = previous->Position - ((ulong)m_Buffer),
|
||||
currHead = current->Position - ((ulong)m_Buffer);
|
||||
return currHead / nextLevelSize == prevHead / nextLevelSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determine segment index based on size and originLevel
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int DetermineSegmentIndex(int[] levelSizeArray, long size, int level)
|
||||
private static int DetermineSegmentIndex(
|
||||
uint[] levelSizeArray,
|
||||
ulong size,
|
||||
int level
|
||||
)
|
||||
{
|
||||
level = int.Min(level, MaxLevel);
|
||||
return (int)((size - 1) / levelSizeArray[level]);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void InsertEntryAfter(Entry* entry, Entry* position)
|
||||
private static void InsertEntryAfter(
|
||||
Entry* entry,
|
||||
Entry* position
|
||||
)
|
||||
{
|
||||
Entry* next = position->NextNear;
|
||||
entry->NextNear = next;
|
||||
@@ -78,7 +92,10 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void InsertEntryBefore(Entry* entry, Entry* position)
|
||||
private static void InsertEntryBefore(
|
||||
Entry* entry,
|
||||
Entry* position
|
||||
)
|
||||
{
|
||||
Entry* prev = position->PreviousNear;
|
||||
entry->NextNear = position;
|
||||
@@ -92,24 +109,26 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// Current Entry will serve as the final merged block. The Merge will be in lazy
|
||||
/// strategy.
|
||||
/// </summary>
|
||||
private void MergeAndRemoveFromBuckets(Entry* entry)
|
||||
private void MergeAndRemoveFromBuckets(
|
||||
Entry* entry
|
||||
)
|
||||
{
|
||||
if (entry == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
long size = entry->Size;
|
||||
nuint size = entry->Size;
|
||||
int originLevel = entry->Level;
|
||||
int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
|
||||
do
|
||||
{
|
||||
// previous direction merge
|
||||
Entry* p = entry->PreviousNear;
|
||||
if (p == null ||
|
||||
p == _spareMemory ||
|
||||
p->State != EntryState.Empty ||
|
||||
p->Level != entry->Level ||
|
||||
if ((p == null) ||
|
||||
(p == _spareMemory) ||
|
||||
(p->State != EntryState.Empty) ||
|
||||
(p->Level != entry->Level) ||
|
||||
!AssertBoundary(p, entry, originLevel))
|
||||
{
|
||||
break;
|
||||
@@ -126,10 +145,10 @@ namespace StgSharp.Mathematics.Memory
|
||||
{
|
||||
// next direction merge
|
||||
Entry* n = entry->NextNear;
|
||||
if (n == null ||
|
||||
n == _spareMemory ||
|
||||
n->State != EntryState.Empty ||
|
||||
n->Level != entry->Level ||
|
||||
if ((n == null) ||
|
||||
(n == _spareMemory) ||
|
||||
(n->State != EntryState.Empty) ||
|
||||
(n->Level != entry->Level) ||
|
||||
!AssertBoundary(n, entry, originLevel))
|
||||
{
|
||||
break;
|
||||
@@ -141,7 +160,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
RemoveFromPositionChain(n);
|
||||
} while (true);
|
||||
originLevel = int.Min(9, originLevel);
|
||||
entry->Level += entry->Size >= _levelSizeArray[originLevel] * 4 ? 1 : 0;
|
||||
entry->Level += (entry->Size >= _levelSizeArray[originLevel] * 4) ? 1 : 0;
|
||||
entry->Level &= 15;
|
||||
|
||||
BucketNode* bucket = (BucketNode*)entry->Position;
|
||||
@@ -152,7 +171,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// Remove node from position linked list, this method does not recycle a BucketNode from
|
||||
/// size bucket
|
||||
/// </summary>
|
||||
private void RemoveFromPositionChain(Entry* entry)
|
||||
private void RemoveFromPositionChain(
|
||||
Entry* entry
|
||||
)
|
||||
{
|
||||
Entry* prev = entry->PreviousNear;
|
||||
Entry* next = entry->NextNear;
|
||||
@@ -169,7 +190,11 @@ namespace StgSharp.Mathematics.Memory
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static void SetEntryLevel(Entry* entry, nuint position, long size)
|
||||
private static void SetEntryLevel(
|
||||
Entry* entry,
|
||||
nuint position,
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
entry ->Position = position;
|
||||
entry->Size = size;
|
||||
@@ -188,7 +213,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
|
||||
|
||||
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
|
||||
[FieldOffset(16)] internal long Size; // Block size information
|
||||
[FieldOffset(16)] internal nuint Size; // Block size information
|
||||
[FieldOffset(36)] public EntryState State;
|
||||
[FieldOffset(28)] public nuint Position; // Memory block position
|
||||
|
||||
|
||||
@@ -1,30 +1,27 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="HLSFAllocator.cs"
|
||||
// file="HLSFAllocator"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -33,9 +30,9 @@ using System.Buffers;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
|
||||
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
@@ -50,32 +47,36 @@ namespace StgSharp.Mathematics.Memory
|
||||
private readonly Action<nuint> _freeHandle;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly long _maxSize;
|
||||
private readonly nuint _maxSize;
|
||||
private readonly nuint _size;
|
||||
private readonly SlabAllocator<Entry> _entries;
|
||||
|
||||
internal HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
long maxBlockSize,
|
||||
nuint maxBlockSize,
|
||||
Func<nuint, int, nuint> allocator,
|
||||
Action<nuint> freeHandle,
|
||||
SlabAllocator<Entry>? entryManager)
|
||||
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle, entryManager) { }
|
||||
SlabAllocator<Entry>? entryManager
|
||||
)
|
||||
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle,
|
||||
entryManager) { }
|
||||
|
||||
internal HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
long maxBlockSize,
|
||||
nuint maxBlockSize,
|
||||
void* externalPointer,
|
||||
Action<nuint> freeHandle,
|
||||
SlabAllocator<Entry>? entryManager = null)
|
||||
SlabAllocator<Entry>? entryManager = null
|
||||
)
|
||||
{
|
||||
if (align < 16 || (align & (align - 1)) != 0) {
|
||||
if ((align < 16) || ((align & (align - 1)) != 0)) {
|
||||
throw new ArgumentException(
|
||||
"Alignment must be a power of two and at least 16 bytes.");
|
||||
}
|
||||
_entries = entryManager ?? SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
|
||||
_entries = entryManager ??
|
||||
SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
|
||||
m_Buffer = (byte*)externalPointer;
|
||||
_align = align;
|
||||
_size = byteSize;
|
||||
@@ -85,13 +86,13 @@ namespace StgSharp.Mathematics.Memory
|
||||
_spareMemory->PreviousNear = _spareMemory;
|
||||
_spareMemory->NextNear = _spareMemory;
|
||||
_spareMemory->Position = (nuint)m_Buffer;
|
||||
_spareMemory->Size = (long)byteSize;
|
||||
_spareMemory->Size = byteSize;
|
||||
|
||||
int levelCountOrigin = _levelSizeArray.Length;
|
||||
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
|
||||
Array.Resize(ref _levelSizeArray, levelCount);
|
||||
_bucketHeads = new BucketNode*[(levelCount * 3) + 1];
|
||||
int size = _levelSizeArray[^1];
|
||||
uint size = _levelSizeArray[^1];
|
||||
for (int i = levelCountOrigin; i < levelCount; i++)
|
||||
{
|
||||
size *= 4;
|
||||
@@ -101,22 +102,37 @@ namespace StgSharp.Mathematics.Memory
|
||||
_freeHandle = freeHandle ?? (static _ => { });
|
||||
}
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize)
|
||||
: this(
|
||||
byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
public HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize
|
||||
)
|
||||
: this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)
|
||||
: this(
|
||||
byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
public HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
uint maxBlockSize
|
||||
)
|
||||
: this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
|
||||
: this(
|
||||
byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
public HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align
|
||||
)
|
||||
: this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align, long maxBlockSize)
|
||||
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
public HybridLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
nuint maxBlockSize
|
||||
)
|
||||
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public partial hlsfHandle Alloc(long size);
|
||||
public partial HlsfHandle Alloc(
|
||||
nuint size
|
||||
);
|
||||
|
||||
public partial void Collect();
|
||||
|
||||
@@ -126,7 +142,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(bool disposing)
|
||||
protected virtual void Dispose(
|
||||
bool disposing
|
||||
)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
|
||||
@@ -44,13 +44,23 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private unsafe struct CacheLineHead
|
||||
{
|
||||
|
||||
internal int _id;
|
||||
internal int _predictor;
|
||||
internal int _refCount;
|
||||
internal nuint _address;
|
||||
internal nuint _origin;
|
||||
internal uint _profile;
|
||||
|
||||
#region
|
||||
|
||||
internal int _refCount;
|
||||
internal uint _size;
|
||||
|
||||
#endregion
|
||||
|
||||
#region
|
||||
|
||||
internal int _id;
|
||||
internal short _predictor;
|
||||
internal ushort _profile;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
/**/
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
@@ -57,6 +58,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
public unsafe DebugSnapshot GetDebugSnapshot()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
/*
|
||||
int maxId = _maxId;
|
||||
byte[] rrpv = new byte[maxId];
|
||||
byte* p = (byte*)_eviction;
|
||||
@@ -64,6 +67,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
rrpv[i] = p[i];
|
||||
}
|
||||
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
|
||||
/**/
|
||||
}
|
||||
|
||||
public readonly struct DebugSnapshot
|
||||
@@ -104,4 +108,5 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
|
||||
}
|
||||
/**/
|
||||
}
|
||||
|
||||
@@ -34,6 +34,8 @@ using System.Runtime.Intrinsics.X86;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
@@ -78,7 +80,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
// EvictLine(i + index);
|
||||
_ = _map.TryRemove(head->_origin, out _);
|
||||
int id = (int)head->_predictor;
|
||||
int id = head->_predictor;
|
||||
IL4Predict predict = pSpan[id] ??
|
||||
throw new L4PredictUnregisteredException(id);
|
||||
predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
|
||||
@@ -97,23 +99,29 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RecycleLine(
|
||||
int id
|
||||
private unsafe void RecycleLine(
|
||||
int id
|
||||
)
|
||||
{
|
||||
_idMux.Push(id);
|
||||
Hlsf.Entry* entry = _entryManager._basePtr + id;
|
||||
_bufferAllocator.Free(entry);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool TryGetEmptyLine(
|
||||
out int index
|
||||
private unsafe bool TryGetEmptyLine(
|
||||
nuint size,
|
||||
out int index,
|
||||
out HlsfHandle handle
|
||||
)
|
||||
{
|
||||
if (_idMux.TryPop(out index)) {
|
||||
return true;
|
||||
if (!_bufferAllocator.TryAlloc(size, out handle))
|
||||
{
|
||||
index = 0;
|
||||
return false;
|
||||
}
|
||||
index = _maxId;
|
||||
return Interlocked.Increment(ref _maxId) <= CacheLineCount;
|
||||
Hlsf.Entry* entry = handle.EntryHandle;
|
||||
index = (int)(entry - _entryManager._basePtr);
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Common.Collections;
|
||||
using StgSharp.Mathematics.Memory;
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
@@ -61,6 +61,9 @@ namespace StgSharp.HighPerformance.Memory
|
||||
if (!_idReuse.TryPop(out int id)) {
|
||||
id = _largestId++;
|
||||
}
|
||||
if (id > short.MaxValue) {
|
||||
throw new OverflowException("Too many predictors in waiting");
|
||||
}
|
||||
_predictors[id] = predict;
|
||||
_predictionQueue.Enqueue(id);
|
||||
return id;
|
||||
@@ -90,8 +93,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
if (_predictionQueue.TryDequeue(out int id))
|
||||
{
|
||||
IL4Predict predict = _predictors[id] ??
|
||||
throw new L4PredictUnregisteredException(id);
|
||||
IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id);
|
||||
CurrentPredict = predict;
|
||||
CurrentPredictIndex = id;
|
||||
return true;
|
||||
|
||||
@@ -71,7 +71,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
/// to the current predictor.
|
||||
/// </remarks>
|
||||
bool IsSamePolicy(
|
||||
uint policy,
|
||||
ushort policy,
|
||||
IL4Predict predict
|
||||
);
|
||||
|
||||
@@ -161,7 +161,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
#pragma warning disable CA1051
|
||||
public nuint Address;
|
||||
public int Size;
|
||||
public uint Size;
|
||||
public ushort MapPolicy;
|
||||
internal short Predictor;
|
||||
#pragma warning restore CA1051
|
||||
|
||||
@@ -60,7 +60,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
ref CacheLinePrediction handle = ref span[i];
|
||||
nuint address = handle.Address;
|
||||
if (!TestHit(address))
|
||||
if (!_map.TryGet(address, out _))
|
||||
{
|
||||
// miss and record
|
||||
span[missIndex] = handle;
|
||||
@@ -88,19 +88,21 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
int predictId = head->_predictor;
|
||||
IL4Predict belong = predictorSpan[predictId];
|
||||
|
||||
// TODO 由于L4的缓存行改为“可变长度”缓存行,需要调整缓存行复用策略
|
||||
// This branch performs resident-line adoption rather than a simple policy
|
||||
// equality check. Returning true means the current predictor can keep using
|
||||
// the existing cache bytes without a write-back plus re-prefetch cycle.
|
||||
if (predict.IsSamePolicy(head->_profile, belong))
|
||||
{
|
||||
head->_predictor =predictIndex;
|
||||
head->_predictor = (short)predictIndex;
|
||||
} else
|
||||
{
|
||||
// Compatibility failed, so the old owner must first flush its view of
|
||||
// the line before the new predictor rematerializes the cache bytes.
|
||||
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
|
||||
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, head->_size));
|
||||
head->_profile = pHandle.MapPolicy;
|
||||
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
|
||||
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, head->_size));
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -110,7 +112,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
int retryCount = 0, remain = missIndex - i;
|
||||
const int maxRetry = 1024;
|
||||
while (!TryGetEmptyLine(out index))
|
||||
HlsfHandle hlsfHandle;
|
||||
while (!TryGetEmptyLine(pHandle.Size, out index, out hlsfHandle))
|
||||
{
|
||||
remain = Evict(remain);
|
||||
retryCount++;
|
||||
@@ -126,11 +129,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
head->_origin = address;
|
||||
head->_profile = pHandle.MapPolicy;
|
||||
head->_refCount = 0;
|
||||
head->_predictor = predictIndex;
|
||||
head->_address = (nuint)(_data + index);
|
||||
head->_predictor = (short)predictIndex;
|
||||
head->_address = (nuint)hlsfHandle.Pointer;
|
||||
head->_size = pHandle.Size;
|
||||
|
||||
// pre fetch them here
|
||||
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
|
||||
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, (int)head->_size));
|
||||
_ = _map.TryAddOrSet(address, (nuint)head, out _);
|
||||
}
|
||||
_ = Interlocked.Add(ref _aheadCount, actualCount);
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="L4.Slab"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private sealed unsafe class Slab : SlabAllocator<Hlsf.Entry>
|
||||
{
|
||||
|
||||
private const int Capacity = L4.CacheLineCount;
|
||||
|
||||
private int _maxUnused;
|
||||
private int _recycleCount;
|
||||
|
||||
internal readonly Hlsf.Entry* _basePtr;
|
||||
internal readonly int* _indexReuse;
|
||||
|
||||
public Slab(
|
||||
nuint ptr
|
||||
)
|
||||
{
|
||||
_basePtr = (Hlsf.Entry*)ptr;
|
||||
_indexReuse = (int*)(_basePtr + Capacity);
|
||||
_maxUnused = 0;
|
||||
_recycleCount = 0;
|
||||
}
|
||||
|
||||
public static nuint RequestedAllocation { get; } = ((nuint)(Unsafe.SizeOf<Hlsf.Entry>() + sizeof(int))) * Capacity;
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycleCount > 0) {
|
||||
return (nuint)(_basePtr + _indexReuse[_recycleCount - 1]);
|
||||
}
|
||||
if (_maxUnused >= Capacity) {
|
||||
throw new OverflowException("Slab is full");
|
||||
}
|
||||
int index = _maxUnused;
|
||||
_maxUnused++;
|
||||
return (nuint)(_basePtr + index);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_indexReuse[_recycleCount] = (int)index;
|
||||
_recycleCount++;
|
||||
}
|
||||
|
||||
public override SlabAllocationHandle<Hlsf.Entry> ReadAllocation()
|
||||
{
|
||||
return new(this, false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -1,40 +1,37 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="L4.cs"
|
||||
// file="L4"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.Mathematics.Memory;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System.Data;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
using HLSF = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
|
||||
using HLSF = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
@@ -42,6 +39,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
|
||||
private const int CacheLineCount = 8192;
|
||||
private const nuint PageSize = 4096;
|
||||
private const int PredictionCount = 2048;
|
||||
private const int PredictionLeastAhead = 1024;
|
||||
|
||||
@@ -58,54 +56,62 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private int _aheadCount;
|
||||
|
||||
private readonly int _bindedNuma;
|
||||
private int _maxId;
|
||||
private volatile int _prefetching;
|
||||
private readonly nuint _baseAddress;
|
||||
private readonly SlabAllocator<HLSF.Entry> _entryManager;
|
||||
private readonly Slab _entryManager;
|
||||
private readonly SwissTable _map;
|
||||
private readonly UnmanagedStack<int> _idMux;
|
||||
|
||||
public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, int maxCacheLineSize = 8192)
|
||||
public L4(
|
||||
int bindedNuma,
|
||||
nuint bufferSize = 8192 * 8192,
|
||||
int align = 16,
|
||||
nuint maxCacheLineSize = 8192
|
||||
)
|
||||
{
|
||||
_bindedNuma = bindedNuma;
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint cacheSize = ((nuint)Unsafe.SizeOf<HLSF.Entry>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
|
||||
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
|
||||
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
|
||||
nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
|
||||
ByteCapacity = normalizedBufferSize;
|
||||
|
||||
// malloc all memory at once, and free at once in Dispose.
|
||||
// This is to avoid fragmentation and improve performance.
|
||||
_baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
|
||||
_baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
|
||||
nuint ptr = _baseAddress;
|
||||
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
_entryManager = SlabAllocator.FromBuffer();
|
||||
AllocSize = entrySize;
|
||||
|
||||
_bufferAllocator = new HLSF(cacheSize, align, maxCacheLineSize, (void*)_baseAddress, static _ => { });
|
||||
nuint _buffer = _baseAddress;
|
||||
ptr += cacheSize;
|
||||
_head = (CacheLineHead*)ptr;
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
nuint _buffer = _baseAddress + normalizedBufferSize;
|
||||
_entryManager = new Slab(_buffer);
|
||||
_bufferAllocator = new HLSF(entrySize, align, maxCacheLineSize, (void*)_baseAddress,
|
||||
static _ => { }, _entryManager);
|
||||
ptr += entrySize;
|
||||
_head = (CacheLineHead*)ptr;
|
||||
ptr += headSize;
|
||||
_prediction = (CacheLinePrediction*)ptr;
|
||||
ptr += predictionSize;
|
||||
_map = SwissTable.Create(ptr, _ => { });
|
||||
ptr += mapSize;
|
||||
_eviction = (EvictionArray*)ptr;
|
||||
ptr += rrpvSize;
|
||||
_idMux = new UnmanagedStack<int>((int*)ptr, CacheLineCount, _ => { });
|
||||
ptr += stackSize;
|
||||
}
|
||||
|
||||
public static nuint AllocSize { get; }
|
||||
public nuint ByteCapacity { get; init; }
|
||||
|
||||
public nuint AllocSize { get; init; }
|
||||
|
||||
// public IL4Predict PrefetchPredict { get; set; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ContinueOnce(L4CacheLine line)
|
||||
public void ContinueOnce(
|
||||
L4CacheLine line
|
||||
)
|
||||
{
|
||||
if (line.Address > 0) {
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
@@ -127,22 +133,25 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
_disposed = true;
|
||||
_map.Dispose();
|
||||
_idMux.Dispose();
|
||||
_entryManager.Dispose();
|
||||
_bufferAllocator.Dispose();
|
||||
|
||||
|
||||
nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
|
||||
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
|
||||
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
|
||||
nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
|
||||
Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
|
||||
nuint normalizedBufferSize = ByteCapacity; // align to 4096 byte, page size
|
||||
Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public L4CacheLine Map(nuint origin)
|
||||
public L4CacheLine Map(
|
||||
nuint origin
|
||||
)
|
||||
{
|
||||
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
|
||||
{
|
||||
@@ -194,7 +203,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private ref int _refCount;
|
||||
private nuint _handle;
|
||||
|
||||
internal L4CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount)
|
||||
internal L4CacheLine(
|
||||
nuint handle,
|
||||
ref int activeCount,
|
||||
ref int cacheRefCount
|
||||
)
|
||||
{
|
||||
_handle = handle;
|
||||
_refCount = ref cacheRefCount;
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocationHandle"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -28,14 +28,17 @@
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe struct SlabAllocationHandle<T> : IDisposable where T: unmanaged
|
||||
public unsafe struct SlabAllocationHandle<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
private readonly SlabAllocator<T> _allocator;
|
||||
|
||||
internal SlabAllocationHandle(SlabAllocator<T> allocator, bool locked)
|
||||
internal SlabAllocationHandle(
|
||||
SlabAllocator<T> allocator,
|
||||
bool locked
|
||||
)
|
||||
{
|
||||
_allocator = allocator;
|
||||
Locked = locked;
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.Chunked"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -33,9 +33,9 @@ using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
|
||||
{
|
||||
|
||||
private bool _disposed;
|
||||
@@ -47,9 +47,11 @@ namespace StgSharp.Mathematics.Memory
|
||||
private nuint _currentBuffer;
|
||||
private nuint _currentCapacity, _currentIndex;
|
||||
|
||||
public ChunkedSlabAllocator(nuint count)
|
||||
public ChunkedSlabAllocator(
|
||||
nuint count
|
||||
)
|
||||
{
|
||||
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
|
||||
_currentBuffer = (nuint)NativeMemory.Alloc(count * ((nuint)_elementSize));
|
||||
|
||||
_currentCapacity = count;
|
||||
int initCount = count switch
|
||||
@@ -60,7 +62,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _currentBuffer + (nuint)(i * _elementSize);
|
||||
span[i] = _currentBuffer + ((nuint)(i * _elementSize));
|
||||
}
|
||||
_buffers = new(4);
|
||||
_buffers.Push(_currentBuffer);
|
||||
@@ -80,14 +82,14 @@ namespace StgSharp.Mathematics.Memory
|
||||
if (curIndex >= _currentCapacity)
|
||||
{
|
||||
nuint cap = _currentCapacity;
|
||||
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
|
||||
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * ((nuint)_elementSize));
|
||||
_buffers.Push(newBuffer);
|
||||
_currentBuffer = newBuffer;
|
||||
_currentIndex = 0;
|
||||
}
|
||||
curIndex = _currentIndex;
|
||||
_currentIndex += 1;
|
||||
return _currentBuffer + ((nuint)_elementSize * curIndex);
|
||||
return _currentBuffer + (((nuint)_elementSize) * curIndex);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
@@ -114,7 +116,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(nuint index)
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
{
|
||||
|
||||
@@ -1,35 +1,32 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.Fixed.cs"
|
||||
// file="SlabAllocator.Fixed"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.Mathematics.Memory;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using StgSharp.Threading;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
@@ -40,7 +37,7 @@ using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
|
||||
{
|
||||
|
||||
private Action<nuint> _freeHandle;
|
||||
@@ -53,7 +50,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
private nuint _Capacity, _currentIndex;
|
||||
|
||||
internal FixedSlabAllocator(nuint buffer, nuint count, Action<nuint> freeHandle)
|
||||
internal FixedSlabAllocator(
|
||||
nuint buffer,
|
||||
nuint count,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
_freeHandle = freeHandle;
|
||||
_buffer = buffer;
|
||||
@@ -67,7 +68,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _buffer + (nuint)(i * _elementSize);
|
||||
span[i] = _buffer + ((nuint)(i * _elementSize));
|
||||
}
|
||||
_recycle = CapacityFixedStackBuilder.Create(span);
|
||||
_Capacity = count;
|
||||
@@ -87,7 +88,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
curIndex = _currentIndex;
|
||||
_currentIndex += 1;
|
||||
return _buffer + ((nuint)_elementSize * curIndex);
|
||||
return _buffer + (((nuint)_elementSize) * curIndex);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
@@ -108,7 +109,9 @@ namespace StgSharp.HighPerformance.Memory
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(nuint index)
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.SequencialConcurrent"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
@@ -32,9 +32,10 @@ using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
|
||||
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T>
|
||||
where T : unmanaged
|
||||
{
|
||||
|
||||
private T* _buffer;
|
||||
@@ -47,18 +48,20 @@ namespace StgSharp.Mathematics.Memory
|
||||
private nuint _highestAllocated;
|
||||
private ulong _capacity;
|
||||
|
||||
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
|
||||
public ConcurrentSequentialSlabAllocator(
|
||||
nuint initialCapacity
|
||||
)
|
||||
{
|
||||
_elementSize = Unsafe.SizeOf<T>();
|
||||
_capacity = initialCapacity;
|
||||
_highestAllocated = 0;
|
||||
|
||||
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
|
||||
_buffer = (T*)NativeMemory.Alloc(((nuint)_capacity) * ((nuint)_elementSize));
|
||||
int initCount = _capacity switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)_capacity / 4
|
||||
_ => ((int)_capacity) / 4
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
@@ -68,11 +71,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
_highestAllocated += (nuint)initCount;
|
||||
}
|
||||
|
||||
internal T* BasePointer
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get => _buffer;
|
||||
}
|
||||
internal T* BasePointer => _buffer;
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
@@ -92,7 +91,7 @@ namespace StgSharp.Mathematics.Memory
|
||||
try
|
||||
{
|
||||
_capacity = Volatile.Read(ref _capacity) * 2;
|
||||
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
|
||||
nuint newSize = ((nuint)_capacity) * ((nuint)_elementSize);
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
@@ -144,7 +143,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
|
||||
public override void Free(nuint index)
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_stack.Push(index);
|
||||
}
|
||||
|
||||
@@ -1,37 +1,34 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="SlabAllocator.cs"
|
||||
// file="SlabAllocator"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload Space
|
||||
// Copyright (c) Nitload Space. All rights reserved.
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the “Software”), to deal in the Software without restriction,
|
||||
// including without limitation the rights to use, copy, modify, merge,
|
||||
// publish, distribute, sublicense, and/or sell copies of the Software,
|
||||
// and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and
|
||||
// this permission notice shall be included in all copies
|
||||
// or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED “AS IS”,
|
||||
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
|
||||
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
|
||||
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
// ARISING FROM, OUT OF OR IN CONNECTION WITH
|
||||
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using System;
|
||||
|
||||
namespace StgSharp.Mathematics.Memory
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public static class SlabAllocator
|
||||
{
|
||||
@@ -58,54 +55,62 @@ namespace StgSharp.Mathematics.Memory
|
||||
/// Thrown when requesting unsupported configurations.
|
||||
/// </exception>
|
||||
/// <remarks>
|
||||
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
|
||||
/// /// /// allocators themselves may be designed for single-threaded use, allocated
|
||||
/// objects /// are /// frequently shared across multiple threads in real-world
|
||||
/// applications. Due to /// the /// special implementation of expandable concurrent
|
||||
/// sequential SLAB allocators, /// they rely /// on internal expansion locks to
|
||||
/// ensure that memory buffer expansions /// (which involve /// copying the entire
|
||||
/// buffer to a new location) do not occur while /// other threads are /// reading
|
||||
/// from allocated regions. </para> <para><strong>/// Sequential Layout Limitations:
|
||||
/// ///</strong><br /> Sequential layout SLAB allocators /// cannot provide a single-
|
||||
/// threaded /// version because:<list /// type="bullet"><item><description>They use
|
||||
/// <para><strong> Important Threading Considerations: </strong></para> <para> While SLAB
|
||||
/// /// /// /// allocators themselves may be designed for single-threaded use,
|
||||
/// allocated /// objects /// are /// frequently shared across multiple threads in
|
||||
/// real-world /// applications. Due to /// the /// special implementation of
|
||||
/// expandable concurrent /// sequential SLAB allocators, /// they rely /// on
|
||||
/// internal expansion locks to /// ensure that memory buffer expansions /// (which
|
||||
/// involve /// copying the entire /// buffer to a new location) do not occur while
|
||||
/// /// other threads are /// reading /// from allocated regions.</para>
|
||||
/// <para><strong> /// Sequential Layout Limitations: /// /// </strong><br />
|
||||
/// Sequential layout SLAB allocators /// cannot provide a single- /// threaded ///
|
||||
/// version because:<list /// type="bullet"><item><description> They use ///
|
||||
/// contiguous memory /// layout that requires /// buffer expansion when capacity is
|
||||
/// exceeded /// ///</description></item><item><description>Buffer expansion involves
|
||||
/// copying all /// /// existing data to a new memory
|
||||
/// location</description></item><item><description> /// /// Without proper
|
||||
/// /// exceeded /// /// </description></item><item><description> Buffer expansion
|
||||
/// involves /// copying all /// /// existing data to a new memory /// location
|
||||
/// </description></item><item><description>/// /// Without proper ///
|
||||
/// synchronization, concurrent access to allocated objects during /// /// expansion
|
||||
/// would cause memory corruption</description></item><item><description>The /// ///
|
||||
/// expansion lock mechanism is essential even if allocation itself is single- /// ///
|
||||
/// threaded </description></item></list></para> <para><strong>Chunked Layout: ///
|
||||
/// ///</strong><br /> Chunked layout allocators can optionally provide single-threaded
|
||||
/// /// /// versions since they don't require buffer expansion and copying. Each
|
||||
/// chunk is /// /// allocated independently, avoiding the memory relocation issues
|
||||
/// present in /// sequential /// layouts. </para>
|
||||
/// /// would cause memory corruption </description></item><item><description> The ///
|
||||
/// /// /// expansion lock mechanism is essential even if allocation itself is
|
||||
/// single- /// /// /// threaded</description></item></list></para> <para><strong>
|
||||
/// Chunked Layout: /// /// /// </strong><br /> Chunked layout allocators can
|
||||
/// optionally provide single-threaded /// /// /// versions since they don't
|
||||
/// require buffer expansion and copying. Each /// chunk is /// /// allocated
|
||||
/// independently, avoiding the memory relocation issues /// present in ///
|
||||
/// sequential /// layouts.</para>
|
||||
/// </remarks>
|
||||
public static SlabAllocator<T> Create<T>(nuint count, SlabBufferLayout layout, bool concurrentSupport = true)
|
||||
where T: unmanaged
|
||||
public static SlabAllocator<T> Create<T>(
|
||||
nuint count,
|
||||
SlabBufferLayout layout,
|
||||
bool concurrentSupport = true
|
||||
) where T : unmanaged
|
||||
{
|
||||
return layout switch
|
||||
{
|
||||
SlabBufferLayout.Sequential => concurrentSupport ?
|
||||
new ConcurrentSequentialSlabAllocator<T>(count) :
|
||||
throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."),
|
||||
(new ConcurrentSequentialSlabAllocator<T>(count)) :
|
||||
(throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
|
||||
SlabBufferLayout.Chunked => concurrentSupport ?
|
||||
new ConcurrentChunkedSlabAllocator<T>(count) :
|
||||
new ChunkedSlabAllocator<T>(count),
|
||||
(new ConcurrentChunkedSlabAllocator<T>(count)) :
|
||||
(new ChunkedSlabAllocator<T>(count)),
|
||||
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
|
||||
};
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static SlabAllocator<T> FromBuffer<T>(nuint buffer, nuint count, Action<nuint> freeHandle)
|
||||
where T: unmanaged
|
||||
public static SlabAllocator<T> FromBuffer<T>(
|
||||
nuint buffer,
|
||||
nuint count,
|
||||
Action<nuint> freeHandle
|
||||
) where T : unmanaged
|
||||
{
|
||||
return new FixedSlabAllocator<T>(buffer, count, freeHandle);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public abstract class SlabAllocator<T> : IDisposable where T: unmanaged
|
||||
public abstract class SlabAllocator<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
public abstract nuint Allocate();
|
||||
@@ -116,7 +121,9 @@ namespace StgSharp.Mathematics.Memory
|
||||
|
||||
public abstract void ExitBufferReading();
|
||||
|
||||
public abstract void Free(nuint index);
|
||||
public abstract void Free(
|
||||
nuint index
|
||||
);
|
||||
|
||||
public abstract SlabAllocationHandle<T> ReadAllocation();
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public readonly struct LogicalCore
|
||||
{
|
||||
|
||||
@@ -34,7 +34,7 @@ using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public static partial class Numa
|
||||
{
|
||||
|
||||
@@ -32,7 +32,7 @@ using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public class NumaNode
|
||||
{
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
@@ -25,13 +25,6 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="VariableCapacitySemaphore"
|
||||
// Project: StgSharp.Common
|
||||
// AuthorGroup: Nitload
|
||||
// License: MIT
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
@@ -29,7 +29,7 @@ using StgSharp.Timing;
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.Common.Timing
|
||||
namespace StgSharp.Timing
|
||||
{
|
||||
public abstract class TimeSequence
|
||||
{
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Mathematics.Memory;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Common.Timing;
|
||||
using StgSharp.Timing;
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
@@ -86,18 +86,18 @@ namespace StgSharp.Mathematics.Numeric
|
||||
|
||||
// init thread on all logical cores
|
||||
_numaThreads = Numa.CreateForeachThread<MatrixParallelThread>(
|
||||
core => {
|
||||
MatrixParallelThread thread = new(core);
|
||||
SpinWait.SpinUntil(() => thread.State == MatrixThreadState.Ready);
|
||||
core => {
|
||||
MatrixParallelThread thread = new(core);
|
||||
SpinWait.SpinUntil(() => thread.State == MatrixThreadState.Ready);
|
||||
|
||||
SIMDID simd = thread.Simd;
|
||||
ulong mask = simd.Mask;
|
||||
contextMap[mask] = contextMap.TryGetValue(mask, out int contextCount) ?
|
||||
(contextCount + 1) :
|
||||
1;
|
||||
globalSimd = (globalSimd.Mask != 0) ? UniteSIMDID(globalSimd, simd) : simd;
|
||||
return thread;
|
||||
});
|
||||
SIMDID simd = thread.Simd;
|
||||
ulong mask = simd.Mask;
|
||||
contextMap[mask] = contextMap.TryGetValue(mask, out int contextCount) ?
|
||||
(contextCount + 1) :
|
||||
1;
|
||||
globalSimd = (globalSimd.Mask != 0) ? UniteSIMDID(globalSimd, simd) : simd;
|
||||
return thread;
|
||||
});
|
||||
|
||||
// make global simd context
|
||||
IntrinsicContext globalContext = new();
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using StgSharp.Internal;
|
||||
using StgSharp.Mathematics.Numeric.Runtime;
|
||||
using StgSharp.Threading;
|
||||
|
||||
Reference in New Issue
Block a user