HLSf use nuint rather than long, L4 now uses hlsf rather than array to alloc cache line.

This commit is contained in:
Youmu Liu
2026-04-20 18:10:14 +08:00
parent 1b7c929be4
commit 2e2d64a2ce
38 changed files with 796 additions and 465 deletions
@@ -31,7 +31,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.Collections
namespace StgSharp.Collections
{
public class ConcurrentFlexibleArray<T>(
int capacity = 4
@@ -32,7 +32,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.Collections
namespace StgSharp.Collections
{
/// <summary>
/// Zero-overhead view over a contiguous region of unmanaged memory. Does not own the memory —
@@ -125,7 +125,7 @@ namespace StgSharp.Common.Collections
int length
)
{
Debug.Assert((uint)start + (uint)length <= (uint)Length,
Debug.Assert(((uint)start) + ((uint)length) <= (uint)Length,
"UnmanagedArray.AsSpan range out of bounds");
return new Span<T>(Address + start, length);
}
@@ -144,7 +144,7 @@ namespace StgSharp.Common.Collections
T* ptr
)
{
int offset = (int)(((nint)ptr - (nint)Address) / ElementSize);
int offset = (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
ArgumentOutOfRangeException.ThrowIfNegative(offset, nameof(ptr));
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, Length, nameof(ptr));
return offset;
@@ -169,7 +169,7 @@ namespace StgSharp.Common.Collections
T* ptr
)
{
return (int)(((nint)ptr - (nint)Address) / ElementSize);
return (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
}
}
+2 -2
View File
@@ -25,8 +25,8 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
using StgSharp.Mathematics;
using StgSharp.HighPerformance;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
+4 -3
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="Global"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -31,7 +31,8 @@ global using System;
global using System.Runtime.CompilerServices;
global using System.Threading;
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
[assembly: InternalsVisibleTo("StgSharp.HighPerformance")]
[assembly: InternalsVisibleTo("StgSharp.RegularAnalysis")]
[assembly: InternalsVisibleTo("StgSharp.Graphics")]
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
[assembly: InternalsVisibleTo("StgSharpDebug")]
@@ -29,7 +29,7 @@ using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
public unsafe class BufferStack<T> : IDisposable where T: unmanaged
@@ -33,7 +33,7 @@ using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public static class BufferStackBuilder
{
@@ -104,11 +104,7 @@ namespace StgSharp.Mathematics.Memory
AllocateBuffer(capacity);
}
public ulong Count
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (ulong)Volatile.Read(ref _count);
}
public ulong Count => (ulong)Volatile.Read(ref _count);
public void Clear()
{
@@ -191,7 +187,7 @@ namespace StgSharp.Mathematics.Memory
if (header->Top >= header->Capacity)
{
int newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + ((nuint)newCap * (nuint)sizeof(T));
nuint newSize = ((nuint)sizeof(Header)) + (((nuint)newCap) * ((nuint)sizeof(T)));
_lock.EnterBufferCopy();
try
@@ -291,8 +287,9 @@ namespace StgSharp.Mathematics.Memory
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, (nuint)header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
Buffer.MemoryCopy(sourcePtr, dataStart,
((nuint)header->Capacity) * ((nuint)sizeof(T)),
((nuint)span.Length) * ((nuint)sizeof(T)));
}
header->Top = span.Length;
@@ -303,7 +300,7 @@ namespace StgSharp.Mathematics.Memory
int capacity
)
{
nuint size = (nuint)sizeof(Header) + (nuint)capacity * (nuint)sizeof(T);
nuint size = ((nuint)sizeof(Header)) + (((nuint)capacity) * ((nuint)sizeof(T)));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocationHandle.cs"
// file="HLSFAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -35,17 +32,20 @@ using System.Drawing;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Allocator = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
using Allocator = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public readonly unsafe struct hlsfHandle
public readonly unsafe struct HlsfHandle
{
internal readonly Allocator.Entry* EntryHandle;
public readonly long AllocSize;
public readonly nuint AllocSize;
internal hlsfHandle(Allocator.Entry* handle, long s)
internal HlsfHandle(
Allocator.Entry* handle,
nuint s
)
{
EntryHandle = handle;
Pointer = (byte*)handle->Position;
@@ -54,7 +54,9 @@ namespace StgSharp.Mathematics.Memory
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
}
public readonly ref byte this[uint index]
public readonly ref byte this[
uint index
]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
@@ -66,11 +68,7 @@ namespace StgSharp.Mathematics.Memory
public readonly byte* Pointer { get; init; }
public readonly Span<byte> BufferHandle
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => new(Pointer, (int)AllocSize);
}
public readonly Span<byte> BufferHandle => new(Pointer, (int)AllocSize);
public readonly Enumerator GetEnumerator()
{
@@ -80,11 +78,13 @@ namespace StgSharp.Mathematics.Memory
public ref struct Enumerator : IEnumerator<byte>
{
private hlsfHandle _handle;
private HlsfHandle _handle;
private uint _index = ~0u;
internal Enumerator(hlsfHandle span)
internal Enumerator(
HlsfHandle span
)
{
_handle = span;
Span = span.BufferHandle;
@@ -1,43 +1,43 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Alloc.cs"
// file="HLSFAllocator.Alloc"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Numerics;
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
using System.Reflection.Metadata;
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
public partial hlsfHandle Alloc(long size)
public partial HlsfHandle Alloc(
nuint size
)
{
if (size > _maxSize) {
throw new InvalidOperationException("Requested size exceeds maximum allocatable size.");
@@ -70,18 +70,72 @@ namespace StgSharp.Mathematics.Memory
if (handle != null)
{
int requestedOffset = (segmentIndex + 1) * _levelSizeArray[level];
if (requestedOffset > handle->Size) {
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
if (requestedOffset > (int)handle->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
long remainder = handle->Size - requestedOffset;
nuint remainder = handle->Size - ((nuint)requestedOffset);
SetEntryLevel(handle, handle->Position, (uint)requestedOffset);
handle->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(handle->NextNear, remainder);
}
}
return new hlsfHandle(handle, size);
return new(handle, size);
}
public bool TryAlloc(
nuint size,
out HlsfHandle handle
)
{
if (size > _maxSize)
{
handle = default;
return false;
}
int level = GetLevelFromSize(size);
int segmentIndex = DetermineSegmentIndex(_levelSizeArray, size, level);
int i = level, s = segmentIndex;
Entry* entry;
while (!TryPopLevelForAllocation(i, s, out entry))
{
if (++s >= 3)
{
s = 0;
i++;
}
if (i > MaxLevel)
{
if (TryPopLevelForAllocation(10, 1, out entry))
{
break;
} else if (TryAllocateNew32MBBlock(out entry))
{
break;
}
handle = default;
return false;
}
}
if (entry != null)
{
int requestedOffset = (segmentIndex + 1) * ((int)_levelSizeArray[level]);
if (requestedOffset > (int)entry->Size) {
throw new OverflowException("Out of memory: insufficient block size returned from spare memory.");
}
nuint remainder = entry->Size - ((nuint)requestedOffset);
SetEntryLevel(entry, entry->Position, (uint)requestedOffset);
entry->State = EntryState.Alloc;
if (remainder > 0) {
SliceMemory(entry->NextNear, remainder);
}
}
handle = new(entry, size);
return true;
}
/// <summary>
@@ -94,28 +148,31 @@ namespace StgSharp.Mathematics.Memory
/// <param name="sizeToSlice">
///
/// </param>
private void SliceMemory(Entry* e, long sizeToSlice)
private void SliceMemory(
Entry* e,
nuint sizeToSlice
)
{
Entry* next = e;
long remain = sizeToSlice; // remained size to slice
long offset = (long)e->Position - sizeToSlice - (long)m_Buffer; // offset of buffer to slice
long end = (long)e->Position - (long)m_Buffer; // end offset of buffer to slice
nuint remain = sizeToSlice; // remained size to slice
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
i = int.Min(10, i);
for (; i < _levelSizeArray.Length; i++)
{
int size = _levelSizeArray[i];
int upper = size * 4;
long edge = (offset + upper - 1) / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / size);
uint size = _levelSizeArray[i];
nuint upper = size * 4;
nuint edge = (((offset + upper) - 1U) / upper) * upper;
int count = (int)((nuint.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
nuint newSize = ((nuint)count) * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
@@ -131,17 +188,17 @@ namespace StgSharp.Mathematics.Memory
if (i > MaxLevel)
{
int size = _levelSizeArray[^1];
int upper = size * 4;
long edge = end / upper * upper;
int count = (int)((long.Min(edge, end) - offset) / upper);
uint size = _levelSizeArray[^1];
nuint upper = size * 4;
nuint edge = (end / upper) * upper;
int count = (int)((nuint.Min(edge, end) - offset) / upper);
if (count > 0)
{
long newSize = (long)count * (long)upper;
nuint newSize = ((nuint)count) * upper;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
@@ -157,16 +214,16 @@ namespace StgSharp.Mathematics.Memory
for (i--; i >= 0; i--)
{
int size = _levelSizeArray[i];
long edge = end / size * size;
int count = (int)((long.Min(edge, end) - offset) / size);
uint size = _levelSizeArray[i];
nuint edge = (end / size) * size;
int count = (int)((nuint.Min(edge, end) - offset) / size);
if (count > 0)
{
long newSize = count * size;
nuint newSize = ((nuint)count) * size;
Entry* newEntry = (Entry*)_entries.Allocate();
InsertEntryBefore(newEntry, next);
SetEntryLevel(newEntry, (nuint)offset + (nuint)m_Buffer, newSize);
SetEntryLevel(newEntry, offset + ((nuint)m_Buffer), newSize);
newEntry->State = EntryState.Empty;
BucketNode* b = (BucketNode*)newEntry->Position;
@@ -192,16 +249,18 @@ namespace StgSharp.Mathematics.Memory
/// <returns>
/// True if allocation succeeded
/// </returns>
private bool TryAllocateNew32MBBlock(out Entry* handle)
private bool TryAllocateNew32MBBlock(
out Entry* handle
)
{
nuint pos = _spareMemory->Position;
long remain = _spareMemory->Size;
nuint remain = _spareMemory->Size;
if (remain <= 0)
{
handle = null;
return false;
}
long size = remain > 32 * 1024 * 1024L ? 32 * 1024 * 1024L : remain;
nuint size = (remain > 32 * 1024 * 1024L) ? (32 * 1024 * 1024U) : remain;
// build new entry
@@ -212,7 +271,7 @@ namespace StgSharp.Mathematics.Memory
// set fields of spare memory
_spareMemory->Size -= size;
_spareMemory->Position += (nuint)size;
_spareMemory->Position += size;
handle = e;
@@ -222,7 +281,11 @@ namespace StgSharp.Mathematics.Memory
/// <summary>
/// Helper method for allocation - converts BucketNode to Entry
/// </summary>
private bool TryPopLevelForAllocation(int levelIndex, int sizeIndex, out Entry* entry)
private bool TryPopLevelForAllocation(
int levelIndex,
int sizeIndex,
out Entry* entry
)
{
if (TryPopLevel(levelIndex, sizeIndex, out BucketNode* bucketNode))
{
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="HLSFAllocator.BucketNode"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -30,18 +30,22 @@ using System.Reflection.Metadata;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private void PushBucketToLevel(int levelIndex, int sizeIndex, BucketNode* node)
private void PushBucketToLevel(
int levelIndex,
int sizeIndex,
BucketNode* node
)
{
if (node == null || node->EntryRef == null) {
if ((node == null) || (node->EntryRef == null)) {
return;
}
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
@@ -69,9 +73,13 @@ namespace StgSharp.Mathematics.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
private void RemoveFromBucket(
int levelIndex,
int sizeIndex,
BucketNode* node
)
{
Index index = (levelIndex > MaxLevel) ? ^1 : (3 * levelIndex) + sizeIndex;
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
if (_bucketHeads[index] == node)
{
if (TryPopLevel(levelIndex, sizeIndex, out _)) {
@@ -94,9 +102,13 @@ namespace StgSharp.Mathematics.Memory
return;
}
private bool TryPopLevel(int levelIndex, int sizeIndex, out BucketNode* node)
private bool TryPopLevel(
int levelIndex,
int sizeIndex,
out BucketNode* node
)
{
Index index = levelIndex > MaxLevel ? ^1 : (3 * levelIndex) + sizeIndex;
Index index = (levelIndex > MaxLevel) ? (^1) : ((3 * levelIndex) + sizeIndex);
BucketNode* head = _bucketHeads[index];
@@ -125,9 +137,13 @@ namespace StgSharp.Mathematics.Memory
return true;
}
private bool TryRemoveFromBucket(int levelIndex, int sizeIndex, BucketNode* node)
private bool TryRemoveFromBucket(
int levelIndex,
int sizeIndex,
BucketNode* node
)
{
Index index = levelIndex > 9 ? ^1 : (3 * levelIndex) + sizeIndex;
Index index = (levelIndex > 9) ? (^1) : ((3 * levelIndex) + sizeIndex);
if (_bucketHeads[index] == node)
{
bool ok = TryPopLevel(levelIndex, sizeIndex, out _);
@@ -31,7 +31,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public sealed class HLSFPositionChainBreakException : Exception
{
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.Free.cs"
// file="HLSFAllocator.Free"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -34,13 +31,14 @@ using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Reflection.Emit;
using System.Reflection.Metadata;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public enum FreePolicy
{
@@ -61,8 +59,8 @@ namespace StgSharp.Mathematics.Memory
public partial void Collect()
{
Entry* cur = _spareMemory->NextNear;
long size = 0;
while (cur != null && cur != _spareMemory)
nuint size = 0;
while ((cur != null) && (cur != _spareMemory))
{
// cur is not empty, skip
if (cur->State != EntryState.Empty)
@@ -79,7 +77,7 @@ namespace StgSharp.Mathematics.Memory
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while (cur != null && cur != _spareMemory && cur->State == EntryState.Empty);
} while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
if (size > 0) {
SliceMemory(cur, size);
}
@@ -87,13 +85,16 @@ namespace StgSharp.Mathematics.Memory
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(hlsfHandle handle, FreePolicy collectPolicy = FreePolicy.NoCollect)
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(
HlsfHandle handle,
FreePolicy collectPolicy = FreePolicy.NoCollect
)
{
Entry* e = handle.EntryHandle;
if (e == null) {
@@ -113,7 +114,44 @@ namespace StgSharp.Mathematics.Memory
}
// Calculate final level and segment for the merged block
long finalSize = e->Size;
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
// Console.WriteLine("one block freed");
if (collectPolicy == FreePolicy.FullCollect)
{
Collect();
}
}
internal void Free(
Entry* e,
FreePolicy collectPolicy = FreePolicy.NoCollect
)
{
if (e == null) {
return;
}
if (e->State != EntryState.Alloc) {
return;
}
// Mark as empty so it can be merged
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
// Calculate final level and segment for the merged block
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="HLSFAllocator.Level"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -33,23 +33,38 @@ using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
private const int MaxLevel = 9;
private int[] _levelSizeArray = [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216];
private uint[] _levelSizeArray = [
64U,
256U,
1024U,
4096U,
16384U,
65536U,
262144U,
1048576U,
4194304U,
16777216U
];
public int AlignOfLevel(int bucketLevel)
public int AlignOfLevel(
int bucketLevel
)
{
int blockSize = _levelSizeArray[bucketLevel];
return int.Min(blockSize, _align);
uint blockSize = _levelSizeArray[bucketLevel];
return Math.Min((int)blockSize, _align);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(long size)
private static int GetLevelFromSize(
nuint size
)
{
/*
* Level sizes: [64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, ......]
@@ -72,7 +87,9 @@ namespace StgSharp.Mathematics.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevelFromSize(uint size)
private static int GetLevelFromSize(
uint size
)
{
return GetLevelFromSize(size);
}
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="HLSFAllocator.SegmentEntry"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +29,7 @@ using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
@@ -38,37 +38,51 @@ namespace StgSharp.Mathematics.Memory
/// Check if two Entries are adjacent in memory
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(Entry* current, Entry* other)
private static bool AssertAdjacentTo(
Entry* current,
Entry* other
)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return other->Position + (nuint)other->Size == current->Position ||
current->Position + (nuint)current->Size == other->Position;
return (other->Position + other->Size == current->Position) ||
(current->Position + current->Size == other->Position);
}
/// <summary>
/// check out if a bucket is within the same higher-level boundary as currentOffset
/// </summary>
private bool AssertBoundary(Entry* previous, Entry* current, int level)
private bool AssertBoundary(
Entry* previous,
Entry* current,
int level
)
{
level = int.Min(level, MaxLevel);
int nextLevelSize = _levelSizeArray[level] * 4;
ulong prevHead = previous->Position - (ulong)m_Buffer,
currHead = current->Position - (ulong)m_Buffer;
return (currHead / (ulong)nextLevelSize) == (prevHead / (ulong)nextLevelSize);
uint nextLevelSize = _levelSizeArray[level] * 4;
ulong prevHead = previous->Position - ((ulong)m_Buffer),
currHead = current->Position - ((ulong)m_Buffer);
return currHead / nextLevelSize == prevHead / nextLevelSize;
}
/// <summary>
/// Determine segment index based on size and originLevel
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int DetermineSegmentIndex(int[] levelSizeArray, long size, int level)
private static int DetermineSegmentIndex(
uint[] levelSizeArray,
ulong size,
int level
)
{
level = int.Min(level, MaxLevel);
return (int)((size - 1) / levelSizeArray[level]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryAfter(Entry* entry, Entry* position)
private static void InsertEntryAfter(
Entry* entry,
Entry* position
)
{
Entry* next = position->NextNear;
entry->NextNear = next;
@@ -78,7 +92,10 @@ namespace StgSharp.Mathematics.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void InsertEntryBefore(Entry* entry, Entry* position)
private static void InsertEntryBefore(
Entry* entry,
Entry* position
)
{
Entry* prev = position->PreviousNear;
entry->NextNear = position;
@@ -92,24 +109,26 @@ namespace StgSharp.Mathematics.Memory
/// Current Entry will serve as the final merged block. The Merge will be in lazy
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
private void MergeAndRemoveFromBuckets(
Entry* entry
)
{
if (entry == null) {
return;
}
long size = entry->Size;
nuint size = entry->Size;
int originLevel = entry->Level;
int segment = DetermineSegmentIndex(_levelSizeArray, size, originLevel);
do
{
// previous direction merge
Entry* p = entry->PreviousNear;
if (p == null ||
p == _spareMemory ||
p->State != EntryState.Empty ||
p->Level != entry->Level ||
if ((p == null) ||
(p == _spareMemory) ||
(p->State != EntryState.Empty) ||
(p->Level != entry->Level) ||
!AssertBoundary(p, entry, originLevel))
{
break;
@@ -126,10 +145,10 @@ namespace StgSharp.Mathematics.Memory
{
// next direction merge
Entry* n = entry->NextNear;
if (n == null ||
n == _spareMemory ||
n->State != EntryState.Empty ||
n->Level != entry->Level ||
if ((n == null) ||
(n == _spareMemory) ||
(n->State != EntryState.Empty) ||
(n->Level != entry->Level) ||
!AssertBoundary(n, entry, originLevel))
{
break;
@@ -141,7 +160,7 @@ namespace StgSharp.Mathematics.Memory
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
entry->Level += entry->Size >= _levelSizeArray[originLevel] * 4 ? 1 : 0;
entry->Level += (entry->Size >= _levelSizeArray[originLevel] * 4) ? 1 : 0;
entry->Level &= 15;
BucketNode* bucket = (BucketNode*)entry->Position;
@@ -152,7 +171,9 @@ namespace StgSharp.Mathematics.Memory
/// Remove node from position linked list, this method does not recycle a BucketNode from
/// size bucket
/// </summary>
private void RemoveFromPositionChain(Entry* entry)
private void RemoveFromPositionChain(
Entry* entry
)
{
Entry* prev = entry->PreviousNear;
Entry* next = entry->NextNear;
@@ -169,7 +190,11 @@ namespace StgSharp.Mathematics.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SetEntryLevel(Entry* entry, nuint position, long size)
private static void SetEntryLevel(
Entry* entry,
nuint position,
nuint size
)
{
entry ->Position = position;
entry->Size = size;
@@ -188,7 +213,7 @@ namespace StgSharp.Mathematics.Memory
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(24)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal long Size; // Block size information
[FieldOffset(16)] internal nuint Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(28)] public nuint Position; // Memory block position
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="HLSFAllocator.cs"
// file="HLSFAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -33,9 +30,9 @@ using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using hlsfHandle = StgSharp.Mathematics.Memory.hlsfHandle;
using hlsfHandle = StgSharp.HighPerformance.Memory.HlsfHandle;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator : IDisposable
{
@@ -50,32 +47,36 @@ namespace StgSharp.Mathematics.Memory
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly long _maxSize;
private readonly nuint _maxSize;
private readonly nuint _size;
private readonly SlabAllocator<Entry> _entries;
internal HybridLayerSegregatedFitAllocator(
nuint byteSize,
int align,
long maxBlockSize,
nuint maxBlockSize,
Func<nuint, int, nuint> allocator,
Action<nuint> freeHandle,
SlabAllocator<Entry>? entryManager)
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle, entryManager) { }
SlabAllocator<Entry>? entryManager
)
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle,
entryManager) { }
internal HybridLayerSegregatedFitAllocator(
nuint byteSize,
int align,
long maxBlockSize,
nuint maxBlockSize,
void* externalPointer,
Action<nuint> freeHandle,
SlabAllocator<Entry>? entryManager = null)
SlabAllocator<Entry>? entryManager = null
)
{
if (align < 16 || (align & (align - 1)) != 0) {
if ((align < 16) || ((align & (align - 1)) != 0)) {
throw new ArgumentException(
"Alignment must be a power of two and at least 16 bytes.");
}
_entries = entryManager ?? SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
_entries = entryManager ??
SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte*)externalPointer;
_align = align;
_size = byteSize;
@@ -85,13 +86,13 @@ namespace StgSharp.Mathematics.Memory
_spareMemory->PreviousNear = _spareMemory;
_spareMemory->NextNear = _spareMemory;
_spareMemory->Position = (nuint)m_Buffer;
_spareMemory->Size = (long)byteSize;
_spareMemory->Size = byteSize;
int levelCountOrigin = _levelSizeArray.Length;
int levelCount = GetLevelFromSize(maxBlockSize) + 1;
Array.Resize(ref _levelSizeArray, levelCount);
_bucketHeads = new BucketNode*[(levelCount * 3) + 1];
int size = _levelSizeArray[^1];
uint size = _levelSizeArray[^1];
for (int i = levelCountOrigin; i < levelCount; i++)
{
size *= 4;
@@ -101,22 +102,37 @@ namespace StgSharp.Mathematics.Memory
_freeHandle = freeHandle ?? (static _ => { });
}
public HybridLayerSegregatedFitAllocator(nuint byteSize)
: this(
byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(
nuint byteSize
)
: this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)
: this(
byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(
nuint byteSize,
uint maxBlockSize
)
: this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align)
: this(
byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(
nuint byteSize,
int align
)
: this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(nuint byteSize, int align, long maxBlockSize)
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16), ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public HybridLayerSegregatedFitAllocator(
nuint byteSize,
int align,
nuint maxBlockSize
)
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public partial hlsfHandle Alloc(long size);
public partial HlsfHandle Alloc(
nuint size
);
public partial void Collect();
@@ -126,7 +142,9 @@ namespace StgSharp.Mathematics.Memory
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
{
@@ -44,13 +44,23 @@ namespace StgSharp.HighPerformance.Memory
private unsafe struct CacheLineHead
{
internal int _id;
internal int _predictor;
internal int _refCount;
internal nuint _address;
internal nuint _origin;
internal uint _profile;
#region
internal int _refCount;
internal uint _size;
#endregion
#region
internal int _id;
internal short _predictor;
internal ushort _profile;
#endregion
}
}
@@ -33,6 +33,7 @@ using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
/**/
public partial class L4
{
@@ -57,6 +58,8 @@ namespace StgSharp.HighPerformance.Memory
public unsafe DebugSnapshot GetDebugSnapshot()
{
throw new NotImplementedException();
/*
int maxId = _maxId;
byte[] rrpv = new byte[maxId];
byte* p = (byte*)_eviction;
@@ -64,6 +67,7 @@ namespace StgSharp.HighPerformance.Memory
rrpv[i] = p[i];
}
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
/**/
}
public readonly struct DebugSnapshot
@@ -104,4 +108,5 @@ namespace StgSharp.HighPerformance.Memory
}
}
/**/
}
@@ -34,6 +34,8 @@ using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
@@ -78,7 +80,7 @@ namespace StgSharp.HighPerformance.Memory
{
// EvictLine(i + index);
_ = _map.TryRemove(head->_origin, out _);
int id = (int)head->_predictor;
int id = head->_predictor;
IL4Predict predict = pSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
@@ -97,23 +99,29 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RecycleLine(
int id
private unsafe void RecycleLine(
int id
)
{
_idMux.Push(id);
Hlsf.Entry* entry = _entryManager._basePtr + id;
_bufferAllocator.Free(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool TryGetEmptyLine(
out int index
private unsafe bool TryGetEmptyLine(
nuint size,
out int index,
out HlsfHandle handle
)
{
if (_idMux.TryPop(out index)) {
return true;
if (!_bufferAllocator.TryAlloc(size, out handle))
{
index = 0;
return false;
}
index = _maxId;
return Interlocked.Increment(ref _maxId) <= CacheLineCount;
Hlsf.Entry* entry = handle.EntryHandle;
index = (int)(entry - _entryManager._basePtr);
return true;
}
}
@@ -25,8 +25,8 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Common.Collections;
using StgSharp.Mathematics.Memory;
using StgSharp.Collections;
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
@@ -61,6 +61,9 @@ namespace StgSharp.HighPerformance.Memory
if (!_idReuse.TryPop(out int id)) {
id = _largestId++;
}
if (id > short.MaxValue) {
throw new OverflowException("Too many predictors in waiting");
}
_predictors[id] = predict;
_predictionQueue.Enqueue(id);
return id;
@@ -90,8 +93,7 @@ namespace StgSharp.HighPerformance.Memory
}
if (_predictionQueue.TryDequeue(out int id))
{
IL4Predict predict = _predictors[id] ??
throw new L4PredictUnregisteredException(id);
IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id);
CurrentPredict = predict;
CurrentPredictIndex = id;
return true;
@@ -71,7 +71,7 @@ namespace StgSharp.HighPerformance.Memory
/// to the current predictor.
/// </remarks>
bool IsSamePolicy(
uint policy,
ushort policy,
IL4Predict predict
);
@@ -161,7 +161,7 @@ namespace StgSharp.HighPerformance.Memory
#pragma warning disable CA1051
public nuint Address;
public int Size;
public uint Size;
public ushort MapPolicy;
internal short Predictor;
#pragma warning restore CA1051
@@ -60,7 +60,7 @@ namespace StgSharp.HighPerformance.Memory
{
ref CacheLinePrediction handle = ref span[i];
nuint address = handle.Address;
if (!TestHit(address))
if (!_map.TryGet(address, out _))
{
// miss and record
span[missIndex] = handle;
@@ -88,19 +88,21 @@ namespace StgSharp.HighPerformance.Memory
int predictId = head->_predictor;
IL4Predict belong = predictorSpan[predictId];
// TODO 由于L4的缓存行改为“可变长度”缓存行,需要调整缓存行复用策略
// This branch performs resident-line adoption rather than a simple policy
// equality check. Returning true means the current predictor can keep using
// the existing cache bytes without a write-back plus re-prefetch cycle.
if (predict.IsSamePolicy(head->_profile, belong))
{
head->_predictor =predictIndex;
head->_predictor = (short)predictIndex;
} else
{
// Compatibility failed, so the old owner must first flush its view of
// the line before the new predictor rematerializes the cache bytes.
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan<byte>((byte*)head->_address, head->_size));
head->_profile = pHandle.MapPolicy;
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, head->_size));
}
#endregion
@@ -110,7 +112,8 @@ namespace StgSharp.HighPerformance.Memory
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!TryGetEmptyLine(out index))
HlsfHandle hlsfHandle;
while (!TryGetEmptyLine(pHandle.Size, out index, out hlsfHandle))
{
remain = Evict(remain);
retryCount++;
@@ -126,11 +129,12 @@ namespace StgSharp.HighPerformance.Memory
head->_origin = address;
head->_profile = pHandle.MapPolicy;
head->_refCount = 0;
head->_predictor = predictIndex;
head->_address = (nuint)(_data + index);
head->_predictor = (short)predictIndex;
head->_address = (nuint)hlsfHandle.Pointer;
head->_size = pHandle.Size;
// pre fetch them here
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
predict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, (int)head->_size));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
@@ -0,0 +1,102 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="L4.Slab"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
{
private sealed unsafe class Slab : SlabAllocator<Hlsf.Entry>
{
private const int Capacity = L4.CacheLineCount;
private int _maxUnused;
private int _recycleCount;
internal readonly Hlsf.Entry* _basePtr;
internal readonly int* _indexReuse;
public Slab(
nuint ptr
)
{
_basePtr = (Hlsf.Entry*)ptr;
_indexReuse = (int*)(_basePtr + Capacity);
_maxUnused = 0;
_recycleCount = 0;
}
public static nuint RequestedAllocation { get; } = ((nuint)(Unsafe.SizeOf<Hlsf.Entry>() + sizeof(int))) * Capacity;
public override nuint Allocate()
{
if (_recycleCount > 0) {
return (nuint)(_basePtr + _indexReuse[_recycleCount - 1]);
}
if (_maxUnused >= Capacity) {
throw new OverflowException("Slab is full");
}
int index = _maxUnused;
_maxUnused++;
return (nuint)(_basePtr + index);
}
public override void Dispose()
{
GC.SuppressFinalize(this);
}
public override void EnterBufferReading() { }
public override void ExitBufferReading() { }
public override void Free(
nuint index
)
{
_indexReuse[_recycleCount] = (int)index;
_recycleCount++;
}
public override SlabAllocationHandle<Hlsf.Entry> ReadAllocation()
{
return new(this, false);
}
}
}
}
+64 -51
View File
@@ -1,40 +1,37 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="L4.cs"
// file="L4"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
using StgSharp.Mathematics.Memory;
using StgSharp.HighPerformance.Memory;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
using HLSF = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;
using HLSF = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
@@ -42,6 +39,7 @@ namespace StgSharp.HighPerformance.Memory
{
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
@@ -58,54 +56,62 @@ namespace StgSharp.HighPerformance.Memory
private int _aheadCount;
private readonly int _bindedNuma;
private int _maxId;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SlabAllocator<HLSF.Entry> _entryManager;
private readonly Slab _entryManager;
private readonly SwissTable _map;
private readonly UnmanagedStack<int> _idMux;
public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, int maxCacheLineSize = 8192)
public L4(
int bindedNuma,
nuint bufferSize = 8192 * 8192,
int align = 16,
nuint maxCacheLineSize = 8192
)
{
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint cacheSize = ((nuint)Unsafe.SizeOf<HLSF.Entry>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
_baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
_entryManager = SlabAllocator.FromBuffer();
AllocSize = entrySize;
_bufferAllocator = new HLSF(cacheSize, align, maxCacheLineSize, (void*)_baseAddress, static _ => { });
nuint _buffer = _baseAddress;
ptr += cacheSize;
_head = (CacheLineHead*)ptr;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress + normalizedBufferSize;
_entryManager = new Slab(_buffer);
_bufferAllocator = new HLSF(entrySize, align, maxCacheLineSize, (void*)_baseAddress,
static _ => { }, _entryManager);
ptr += entrySize;
_head = (CacheLineHead*)ptr;
ptr += headSize;
_prediction = (CacheLinePrediction*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapSize;
_eviction = (EvictionArray*)ptr;
ptr += rrpvSize;
_idMux = new UnmanagedStack<int>((int*)ptr, CacheLineCount, _ => { });
ptr += stackSize;
}
public static nuint AllocSize { get; }
public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(L4CacheLine line)
public void ContinueOnce(
L4CacheLine line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
@@ -127,22 +133,25 @@ namespace StgSharp.HighPerformance.Memory
}
_disposed = true;
_map.Dispose();
_idMux.Dispose();
_entryManager.Dispose();
_bufferAllocator.Dispose();
nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
// init order is fallow by align requirement of each component.
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
nuint normalizedBufferSize = ByteCapacity; // align to 4096 byte, page size
Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public L4CacheLine Map(nuint origin)
public L4CacheLine Map(
nuint origin
)
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
@@ -194,7 +203,11 @@ namespace StgSharp.HighPerformance.Memory
private ref int _refCount;
private nuint _handle;
internal L4CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount)
internal L4CacheLine(
nuint handle,
ref int activeCount,
ref int cacheRefCount
)
{
_handle = handle;
_refCount = ref cacheRefCount;
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="SlabAllocationHandle"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -28,14 +28,17 @@
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public unsafe struct SlabAllocationHandle<T> : IDisposable where T: unmanaged
public unsafe struct SlabAllocationHandle<T> : IDisposable where T : unmanaged
{
private readonly SlabAllocator<T> _allocator;
internal SlabAllocationHandle(SlabAllocator<T> allocator, bool locked)
internal SlabAllocationHandle(
SlabAllocator<T> allocator,
bool locked
)
{
_allocator = allocator;
Locked = locked;
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="SlabAllocator.Chunked"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -33,9 +33,9 @@ using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private bool _disposed;
@@ -47,9 +47,11 @@ namespace StgSharp.Mathematics.Memory
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ChunkedSlabAllocator(nuint count)
public ChunkedSlabAllocator(
nuint count
)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentBuffer = (nuint)NativeMemory.Alloc(count * ((nuint)_elementSize));
_currentCapacity = count;
int initCount = count switch
@@ -60,7 +62,7 @@ namespace StgSharp.Mathematics.Memory
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
span[i] = _currentBuffer + ((nuint)(i * _elementSize));
}
_buffers = new(4);
_buffers.Push(_currentBuffer);
@@ -80,14 +82,14 @@ namespace StgSharp.Mathematics.Memory
if (curIndex >= _currentCapacity)
{
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * ((nuint)_elementSize));
_buffers.Push(newBuffer);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _currentBuffer + ((nuint)_elementSize * curIndex);
return _currentBuffer + (((nuint)_elementSize) * curIndex);
}
public override void Dispose()
@@ -114,7 +116,9 @@ namespace StgSharp.Mathematics.Memory
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
public override void Free(
nuint index
)
{
_recycle.Push(index);
}
@@ -36,7 +36,7 @@ using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
{
@@ -1,35 +1,32 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.Fixed.cs"
// file="SlabAllocator.Fixed"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.Mathematics.Memory;
using StgSharp.HighPerformance.Memory;
using StgSharp.Threading;
using System;
using System.Collections.Generic;
@@ -40,7 +37,7 @@ using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private Action<nuint> _freeHandle;
@@ -53,7 +50,11 @@ namespace StgSharp.HighPerformance.Memory
private nuint _Capacity, _currentIndex;
internal FixedSlabAllocator(nuint buffer, nuint count, Action<nuint> freeHandle)
internal FixedSlabAllocator(
nuint buffer,
nuint count,
Action<nuint> freeHandle
)
{
_freeHandle = freeHandle;
_buffer = buffer;
@@ -67,7 +68,7 @@ namespace StgSharp.HighPerformance.Memory
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _buffer + (nuint)(i * _elementSize);
span[i] = _buffer + ((nuint)(i * _elementSize));
}
_recycle = CapacityFixedStackBuilder.Create(span);
_Capacity = count;
@@ -87,7 +88,7 @@ namespace StgSharp.HighPerformance.Memory
}
curIndex = _currentIndex;
_currentIndex += 1;
return _buffer + ((nuint)_elementSize * curIndex);
return _buffer + (((nuint)_elementSize) * curIndex);
}
public override void Dispose()
@@ -108,7 +109,9 @@ namespace StgSharp.HighPerformance.Memory
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(nuint index)
public override void Free(
nuint index
)
{
_recycle.Push(index);
}
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="SlabAllocator.SequencialConcurrent"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -32,9 +32,10 @@ using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T> where T: unmanaged
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T>
where T : unmanaged
{
private T* _buffer;
@@ -47,18 +48,20 @@ namespace StgSharp.Mathematics.Memory
private nuint _highestAllocated;
private ulong _capacity;
public ConcurrentSequentialSlabAllocator(nuint initialCapacity)
public ConcurrentSequentialSlabAllocator(
nuint initialCapacity
)
{
_elementSize = Unsafe.SizeOf<T>();
_capacity = initialCapacity;
_highestAllocated = 0;
_buffer = (T*)NativeMemory.Alloc((nuint)_capacity * (nuint)_elementSize);
_buffer = (T*)NativeMemory.Alloc(((nuint)_capacity) * ((nuint)_elementSize));
int initCount = _capacity switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)_capacity / 4
_ => ((int)_capacity) / 4
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
@@ -68,11 +71,7 @@ namespace StgSharp.Mathematics.Memory
_highestAllocated += (nuint)initCount;
}
internal T* BasePointer
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => _buffer;
}
internal T* BasePointer => _buffer;
public override nuint Allocate()
{
@@ -92,7 +91,7 @@ namespace StgSharp.Mathematics.Memory
try
{
_capacity = Volatile.Read(ref _capacity) * 2;
nuint newSize = (nuint)_capacity * (nuint)_elementSize;
nuint newSize = ((nuint)_capacity) * ((nuint)_elementSize);
_lock.EnterBufferCopy();
try
{
@@ -144,7 +143,9 @@ namespace StgSharp.Mathematics.Memory
_lock.ExitBufferRead();
}
public override void Free(nuint index)
public override void Free(
nuint index
)
{
_stack.Push(index);
}
@@ -1,37 +1,34 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="SlabAllocator.cs"
// file="SlabAllocator"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
namespace StgSharp.Mathematics.Memory
namespace StgSharp.HighPerformance.Memory
{
public static class SlabAllocator
{
@@ -58,54 +55,62 @@ namespace StgSharp.Mathematics.Memory
/// Thrown when requesting unsupported configurations.
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
/// /// /// allocators themselves may be designed for single-threaded use, allocated
/// objects /// are /// frequently shared across multiple threads in real-world
/// applications. Due to /// the /// special implementation of expandable concurrent
/// sequential SLAB allocators, /// they rely /// on internal expansion locks to
/// ensure that memory buffer expansions /// (which involve /// copying the entire
/// buffer to a new location) do not occur while /// other threads are /// reading
/// from allocated regions. </para> <para><strong>/// Sequential Layout Limitations:
/// ///</strong><br /> Sequential layout SLAB allocators /// cannot provide a single-
/// threaded /// version because:<list /// type="bullet"><item><description>They use
/// <para><strong> Important Threading Considerations: </strong></para> <para> While SLAB
/// /// /// /// allocators themselves may be designed for single-threaded use,
/// allocated /// objects /// are /// frequently shared across multiple threads in
/// real-world /// applications. Due to /// the /// special implementation of
/// expandable concurrent /// sequential SLAB allocators, /// they rely /// on
/// internal expansion locks to /// ensure that memory buffer expansions /// (which
/// involve /// copying the entire /// buffer to a new location) do not occur while
/// /// other threads are /// reading /// from allocated regions.</para>
/// <para><strong> /// Sequential Layout Limitations: /// /// </strong><br />
/// Sequential layout SLAB allocators /// cannot provide a single- /// threaded ///
/// version because:<list /// type="bullet"><item><description> They use ///
/// contiguous memory /// layout that requires /// buffer expansion when capacity is
/// exceeded /// ///</description></item><item><description>Buffer expansion involves
/// copying all /// /// existing data to a new memory
/// location</description></item><item><description> /// /// Without proper
/// /// exceeded /// /// </description></item><item><description> Buffer expansion
/// involves /// copying all /// /// existing data to a new memory /// location
/// </description></item><item><description>/// /// Without proper ///
/// synchronization, concurrent access to allocated objects during /// /// expansion
/// would cause memory corruption</description></item><item><description>The /// ///
/// expansion lock mechanism is essential even if allocation itself is single- /// ///
/// threaded </description></item></list></para> <para><strong>Chunked Layout: ///
/// ///</strong><br /> Chunked layout allocators can optionally provide single-threaded
/// /// /// versions since they don't require buffer expansion and copying. Each
/// chunk is /// /// allocated independently, avoiding the memory relocation issues
/// present in /// sequential /// layouts. </para>
/// /// would cause memory corruption </description></item><item><description> The ///
/// /// /// expansion lock mechanism is essential even if allocation itself is
/// single- /// /// /// threaded</description></item></list></para> <para><strong>
/// Chunked Layout: /// /// /// </strong><br /> Chunked layout allocators can
/// optionally provide single-threaded /// /// /// versions since they don't
/// require buffer expansion and copying. Each /// chunk is /// /// allocated
/// independently, avoiding the memory relocation issues /// present in ///
/// sequential /// layouts.</para>
/// </remarks>
public static SlabAllocator<T> Create<T>(nuint count, SlabBufferLayout layout, bool concurrentSupport = true)
where T: unmanaged
public static SlabAllocator<T> Create<T>(
nuint count,
SlabBufferLayout layout,
bool concurrentSupport = true
) where T : unmanaged
{
return layout switch
{
SlabBufferLayout.Sequential => concurrentSupport ?
new ConcurrentSequentialSlabAllocator<T>(count) :
throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion."),
(new ConcurrentSequentialSlabAllocator<T>(count)) :
(throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
SlabBufferLayout.Chunked => concurrentSupport ?
new ConcurrentChunkedSlabAllocator<T>(count) :
new ChunkedSlabAllocator<T>(count),
(new ConcurrentChunkedSlabAllocator<T>(count)) :
(new ChunkedSlabAllocator<T>(count)),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
};
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static SlabAllocator<T> FromBuffer<T>(nuint buffer, nuint count, Action<nuint> freeHandle)
where T: unmanaged
public static SlabAllocator<T> FromBuffer<T>(
nuint buffer,
nuint count,
Action<nuint> freeHandle
) where T : unmanaged
{
return new FixedSlabAllocator<T>(buffer, count, freeHandle);
}
}
public abstract class SlabAllocator<T> : IDisposable where T: unmanaged
public abstract class SlabAllocator<T> : IDisposable where T : unmanaged
{
public abstract nuint Allocate();
@@ -116,7 +121,9 @@ namespace StgSharp.Mathematics.Memory
public abstract void ExitBufferReading();
public abstract void Free(nuint index);
public abstract void Free(
nuint index
);
public abstract SlabAllocationHandle<T> ReadAllocation();
@@ -31,7 +31,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public readonly struct LogicalCore
{
@@ -34,7 +34,7 @@ using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public static partial class Numa
{
@@ -32,7 +32,7 @@ using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Common.HighPerformance.ProcessorAbstraction
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public class NumaNode
{
+1 -1
View File
@@ -25,7 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
@@ -25,13 +25,6 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="VariableCapacitySemaphore"
// Project: StgSharp.Common
// AuthorGroup: Nitload
// License: MIT
// -----------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Threading;
+1 -1
View File
@@ -29,7 +29,7 @@ using StgSharp.Timing;
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.Common.Timing
namespace StgSharp.Timing
{
public abstract class TimeSequence
{
@@ -25,7 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Mathematics.Memory;
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
+1 -1
View File
@@ -25,7 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Common.Timing;
using StgSharp.Timing;
using System;
using System.Runtime.CompilerServices;
using System.Threading;