youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after merge

Browse Source
This commit is contained in:
Nitload
2026-05-10 06:39:52 +08:00
parent 156f6afe85 9234dbe99b
commit 5e73469c2e
52 changed files with 5655 additions and 449 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.Debug.cs
+1 -1
View File
@@ -39,7 +39,7 @@ namespace StgSharp.HighPerformance.Memory
public void DebugAgeAll()
{
AgeAll();
// AgeAll();
}
public bool DebugContains(
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+5 -5
View File
@@ -109,11 +109,11 @@ namespace StgSharp.HighPerformance.Memory
*
* This is safe only with the current L4 allocation/layout guarantees.
*/
short predictCount = Volatile.Read(ref (*_predictCount)[index]);
short mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (short.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_predictCount)[index]), -decrement);
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_mapCount)[index]), -decrement);
int predictCount = Volatile.Read(ref (*_predictCount)[index]);
int mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref (*_predictCount)[index], -decrement);
_ = Interlocked.Add(ref (*_mapCount)[index], -decrement);
}
(*_predictCount)[index]++;
}
StgSharp.Common/HighPerformance/Memory/Ptr64.cs
+384
View File
@@ -0,0 +1,384 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="Ptr64"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
#pragma warning disable CA2225
public readonly struct Ptr64 : IEquatable<Ptr64>, IComparable<Ptr64>, IFormattable
{
public Ptr64(
ulong value
)
{
Value = value;
}
public ulong Value { get; }
public int CompareTo(
Ptr64 other
)
{
return Value.CompareTo(other.Value);
}
public bool Equals(
Ptr64 other
)
{
return Value == other.Value;
}
public override bool Equals(
object? obj
)
{
return obj is Ptr64 other && Equals(other);
}
public override int GetHashCode()
{
return Value.GetHashCode();
}
public override string ToString()
{
return Value.ToString();
}
public string ToString(
string? format,
IFormatProvider? formatProvider
)
{
return Value.ToString(format, formatProvider);
}
public static Ptr64 operator -(
Ptr64 left,
ulong right
)
{
return new(left.Value - right);
}
public static ulong operator -(
Ptr64 left,
Ptr64 right
)
{
return left.Value - right.Value;
}
public static bool operator !=(
Ptr64 left,
Ptr64 right
)
{
return left.Value != right.Value;
}
public static Ptr64 operator &(
Ptr64 left,
ulong mask
)
{
return new(left.Value & mask);
}
public static Ptr64 operator |(
Ptr64 left,
ulong mask
)
{
return new(left.Value | mask);
}
public static Ptr64 operator +(
Ptr64 left,
ulong right
)
{
return new(left.Value + right);
}
public static bool operator <(
Ptr64 left,
Ptr64 right
)
{
return left.Value < right.Value;
}
public static Ptr64 operator <<(
Ptr64 left,
int shift
)
{
return new Ptr64(left.Value << shift);
}
public static bool operator <=(
Ptr64 left,
Ptr64 right
)
{
return left.Value <= right.Value;
}
public static bool operator ==(
Ptr64 left,
Ptr64 right
)
{
return left.Value == right.Value;
}
public static bool operator >(
Ptr64 left,
Ptr64 right
)
{
return left.Value > right.Value;
}
public static bool operator >=(
Ptr64 left,
Ptr64 right
)
{
return left.Value >= right.Value;
}
public static Ptr64 operator >>(
Ptr64 left,
int shift
)
{
return new Ptr64(left.Value >> shift);
}
public static implicit operator Ptr64(
ulong value
)
{
return new(value);
}
public static implicit operator ulong(
Ptr64 value
)
{
return value.Value;
}
}
public readonly struct SPtr64 : IEquatable<SPtr64>, IComparable<SPtr64>, IFormattable
{
public SPtr64(
long value
)
{
Value = value;
}
public long Value { get; }
public int CompareTo(
SPtr64 other
)
{
return Value.CompareTo(other.Value);
}
public bool Equals(
SPtr64 other
)
{
return Value == other.Value;
}
public override bool Equals(
object? obj
)
{
return obj is SPtr64 other && Equals(other);
}
public override int GetHashCode()
{
return Value.GetHashCode();
}
public override string ToString()
{
return Value.ToString();
}
public string ToString(
string? format,
IFormatProvider? formatProvider
)
{
return Value.ToString(format, formatProvider);
}
public static SPtr64 operator -(
SPtr64 left,
long right
)
{
return new(left.Value - right);
}
public static long operator -(
SPtr64 left,
SPtr64 right
)
{
return left.Value - right.Value;
}
public static bool operator !=(
SPtr64 left,
SPtr64 right
)
{
return left.Value != right.Value;
}
public static SPtr64 operator &(
SPtr64 left,
long mask
)
{
return new(left.Value & mask);
}
public static SPtr64 operator |(
SPtr64 left,
long mask
)
{
return new(left.Value | mask);
}
public static SPtr64 operator +(
SPtr64 left,
long right
)
{
return new(left.Value + right);
}
public static bool operator <(
SPtr64 left,
SPtr64 right
)
{
return left.Value < right.Value;
}
public static SPtr64 operator <<(
SPtr64 left,
int shift
)
{
return new SPtr64(left.Value << shift);
}
public static bool operator <=(
SPtr64 left,
SPtr64 right
)
{
return left.Value <= right.Value;
}
public static bool operator ==(
SPtr64 left,
SPtr64 right
)
{
return left.Value == right.Value;
}
public static bool operator >(
SPtr64 left,
SPtr64 right
)
{
return left.Value > right.Value;
}
public static bool operator >=(
SPtr64 left,
SPtr64 right
)
{
return left.Value >= right.Value;
}
public static SPtr64 operator >>(
SPtr64 left,
int shift
)
{
return new SPtr64(left.Value >> shift);
}
public static implicit operator long(
SPtr64 value
)
{
return value.Value;
}
public static implicit operator SPtr64(
long value
)
{
return new(value);
}
}
#pragma warning disable CA2225
}
StgSharp.Common/HighPerformance/Memory/SlabAllocator.cs
+24 -28
View File
@@ -56,29 +56,25 @@ namespace StgSharp.HighPerformance.Memory
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para> While SLAB
/// /// /// /// allocators themselves may be designed for single-threaded use,
/// allocated /// objects /// are /// frequently shared across multiple threads in
/// real-world /// applications. Due to /// the /// special implementation of
/// expandable concurrent /// sequential SLAB allocators, /// they rely /// on
/// internal expansion locks to /// ensure that memory buffer expansions /// (which
/// involve /// copying the entire /// buffer to a new location) do not occur while
/// /// other threads are /// reading /// from allocated regions.</para>
/// <para><strong> /// Sequential Layout Limitations: /// /// </strong><br />
/// Sequential layout SLAB allocators /// cannot provide a single- /// threaded ///
/// version because:<list /// type="bullet"><item><description> They use ///
/// contiguous memory /// layout that requires /// buffer expansion when capacity is
/// /// exceeded /// /// </description></item><item><description> Buffer expansion
/// involves /// copying all /// /// existing data to a new memory /// location
/// </description></item><item><description>/// /// Without proper ///
/// synchronization, concurrent access to allocated objects during /// /// expansion
/// /// would cause memory corruption </description></item><item><description> The ///
/// /// /// expansion lock mechanism is essential even if allocation itself is
/// single- /// /// /// threaded</description></item></list></para> <para><strong>
/// Chunked Layout: /// /// /// </strong><br /> Chunked layout allocators can
/// optionally provide single-threaded /// /// /// versions since they don't
/// require buffer expansion and copying. Each /// chunk is /// /// allocated
/// independently, avoiding the memory relocation issues /// present in ///
/// sequential /// layouts.</para>
/// allocators themselves may be designed for single-threaded use, allocated objects are
/// frequently shared across multiple threads in real-world applications. Due to the
/// special implementation of expandable concurrent sequential SLAB allocators, they rely
/// on internal expansion locks to ensure that memory buffer expansions (which involve
/// copying the entire buffer to a new location) do not occur while other threads are
/// reading from allocated regions.</para> <para><strong>Sequential Layout
/// Limitations:</strong><br /> Sequential layout SLAB allocators cannot provide a single-
/// threaded version because:<list type="bullet"><item><description>They use contiguous
/// memory layout that requires buffer expansion when capacity is
/// exceeded</description></item><item><description>Buffer expansion involves copying all
/// existing data to a new memory location</description></item><item><description> Without
/// proper synchronization, concurrent access to allocated objects during synchronization,
/// concurrent access to allocated objects during expansion would cause memory corruption
/// </description></item><item><description> The expansion lock mechanism is essential
/// even if allocation itself is single- threaded</description></item></list></para>
/// <para><strong> Chunked Layout: </strong><br /> Chunked layout allocators can
/// optionally provide single-threaded versions since they don't require buffer expansion
/// and copying. Each chunk is allocated independently, avoiding the memory relocation
/// issues present in sequential layouts.</para>
/// </remarks>
public static SlabAllocator<T> Create<T>(
nuint count,
@@ -89,12 +85,12 @@ namespace StgSharp.HighPerformance.Memory
return layout switch
{
SlabBufferLayout.Sequential => concurrentSupport ?
(new ConcurrentSequentialSlabAllocator<T>(count)) :
(throw new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
new ConcurrentSequentialSlabAllocator<T>(count) :
throw (new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
SlabBufferLayout.Chunked => concurrentSupport ?
(new ConcurrentChunkedSlabAllocator<T>(count)) :
(new ChunkedSlabAllocator<T>(count)),
_ => throw new ArgumentOutOfRangeException(nameof(layout), layout, null)
new ConcurrentChunkedSlabAllocator<T>(count) :
new ChunkedSlabAllocator<T>(count),
_ => throw (new ArgumentOutOfRangeException(nameof(layout), layout, null))
};
}
StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs
+215
View File
@@ -0,0 +1,215 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="TwoLayerSegregatedFitAllocator"
// Project: StgSharp
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using static StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public unsafe class TwoLayerSegregatedFitAllocator : IDisposable
{
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
private Metadata* _allocEnd;
private readonly Metadata*[] _bucket;
private readonly Metadata* _spareMemory;
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly Ptr64 _basePtr;
private readonly ulong _maxAllocSize;
private readonly ulong _totalSize;
internal TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
ulong maxBlockSize,
Func<nuint, int, nuint> allocator,
Action<nuint> freeHandle
)
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle) { }
internal TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
ulong maxBlockSize,
void* externalPointer,
Action<nuint> freeHandle
)
{
_totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer;
_align = align;
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
_bucket = new Metadata*[58 - BitOperations.LeadingZeroCount(_maxAllocSize)];
_spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
_freeHandle = freeHandle ?? (static_ => { });
_allocEnd = _spareMemory;
// init spare memory node
_spareMemory->Position = 0UL;
_spareMemory->Size = _totalSize - Metadata.SizeOf;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0;
_spareMemory->PreviousNear = 0;
}
public TwoLayerSegregatedFitAllocator(
nuint byteSize
)
: this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
uint maxBlockSize
)
: this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align
)
: this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
nuint maxBlockSize
)
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public Handle Alloc(
nuint size
)
{
ulong allocSize = nuint.Max(64, (~64U) & (size + 63));
if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
}
int level = 57 - BitOperations.LeadingZeroCount(allocSize);
Metadata* entry = null;
for (int i = level; i < _bucket.Length; i++)
{
entry = _bucket[i];
if (entry is not null)
{
_bucket[i] = entry->NextLevel != 0 ? &_spareMemory[-entry->NextLevel] : null;
break;
}
}
if (entry is null)
{
// alloc new from spare
}
ulong remain = entry->Size - allocSize;
throw new NotImplementedException();
}
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
public void Free(
Handle handle
)
{
throw new NotImplementedException();
}
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
{
if (disposing) {
_freeHandle((nuint)_basePtr);
}
disposedValue = true;
}
}
[StructLayout(LayoutKind.Sequential)]
public struct Handle
{
internal readonly Metadata* Meta;
public readonly void* Pointer;
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(24)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
[FieldOffset(22)] public ushort Mask1;
[FieldOffset(30)] public ushort Mask2;
public Ptr64 Position
{
readonly get => (Ptr64)(PositionMask & Mask48Bits);
set => PositionMask = (PositionMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 Size
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
}
}
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.Alloc.cs
+25 -11
View File
@@ -53,13 +53,13 @@ namespace StgSharp.HighPerformance.Memory
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int index = int.Min((3 * level) + segment, _bucketHeads.Length);
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(i, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
if ((i >= _bucketHeads.Length) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException($"Out of memory: unable to satisfy request of {size} B (aligned {requestedOffset} B) from existing buckets or spare memory.");
}
if (entry != null)
{
@@ -96,13 +96,13 @@ namespace StgSharp.HighPerformance.Memory
Entry* entry = null;
nuint requestedOffset = GetAllocSize(size);
(int level, int segment) = GetIndexPairOnce(size);
int index = (3 * level) + segment;
int index = int.Min((3 * level) + segment, _bucketHeads.Length);
int i = index;
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(index, out entry)) {
while ((i < _bucketHeads.Length) && !TryPopLevelForAllocation(i, out entry)) {
i++;
}
if ((i > _maxLevel) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException("Out of memory: Unable to allocate new 16MB block or find suitable bucket for allocation.");
if ((i >= _bucketHeads.Length) && !TryAllocateNewBlock(requestedOffset, out entry)) {
throw new OverflowException($"Out of memory: unable to satisfy request of {size} B (aligned {requestedOffset} B) from existing buckets or spare memory.");
}
if (entry != null)
@@ -138,10 +138,24 @@ namespace StgSharp.HighPerformance.Memory
{
Entry* next = e;
nuint remain = sizeToSlice; // remained size to slice
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
nuint offset = e->Position - sizeToSlice - ((nuint)m_Buffer); // offset of buffer to slice
nuint position = e->Position - sizeToSlice; // position of buffer to slice
nuint end = e->Position - ((nuint)m_Buffer); // end offset of buffer to slice
int i = (BitOperations.TrailingZeroCount(offset) - 6) / 2;
int i = GetBucketIndexOnce(sizeToSlice);
Entry* entry = (Entry*)_entries.Allocate();
SetEntryLevel(entry, position, remain);
entry->State = EntryState.Empty;
BucketNode* bucketNode = (BucketNode*)entry->Position;
bucketNode->EntryRef = entry;
PushNodeToBucket(i, bucketNode);
Entry* prev = next->PreviousNear;
prev->NextNear = entry;
entry->PreviousNear = prev;
entry->NextNear = next;
next->PreviousNear = entry;
}
/// <summary>
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.BucketNode.cs
+12 -11
View File
@@ -34,6 +34,12 @@ namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class HybridLayerSegregatedFitAllocator
{
/*
* All operations with style of (level, segment)
* DO NOT assume that the level are valid.
* They are just used to calculate the bucket index,
* which may exceed the range of _bucketHeads.
*/
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void PushBucketToLevel(
@@ -46,7 +52,7 @@ namespace StgSharp.HighPerformance.Memory
return;
}
int index = (3 * levelIndex) + sizeIndex;
int index = Math.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
PushNodeToBucket(index, node);
}
@@ -55,7 +61,7 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* node
)
{
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
index = Math.Min(index, _bucketHeads.Length - 1);
if (_bucketHeads[index] is null)
{
_bucketHeads[index] = node;
@@ -69,15 +75,12 @@ namespace StgSharp.HighPerformance.Memory
{
BucketNode* head = _bucketHeads[index];
node->NextLevel = head;
BucketNode* headNode = head;
node->IsInBucket = true;
_bucketHeads[index] = node;
// Console.WriteLine($"Push to head of level of {levelIndex} {sizeIndex}, bucket pos {(ulong)node}, previous head is at {head}");
if (headNode != null)
{
headNode->PreviousLevel = node;
}
head->PreviousLevel = node;
return;
}
}
@@ -89,8 +92,7 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* node
)
{
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? _bucketHeads.Length : index;
int index = Math.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
if (_bucketHeads[index] == node)
{
if (TryPopBucket(index, out _)) {
@@ -157,8 +159,7 @@ namespace StgSharp.HighPerformance.Memory
out BucketNode* node
)
{
int index = (3 * levelIndex) + sizeIndex;
index = (index >= _bucketHeads.Length) ? (_bucketHeads.Length - 1) : index;
int index = int.Min((3 * levelIndex) + sizeIndex, _bucketHeads.Length - 1);
return TryPopBucket(index, out node);
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Debug.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.Debug.cs
View File
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Free.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.Free.cs
+15 -19
View File
@@ -74,23 +74,25 @@ namespace StgSharp.HighPerformance.Memory
{
size += cur->Size;
Entry* next = cur->NextNear;
RemoveFromBucket(cur->Level, DetermineSegmentIndex(_levelSizeArray, cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromBucket(cur->Level, GetSegmentFromLevel(cur->Size, cur->Level), (BucketNode*)cur->Position);
RemoveFromPositionChain(cur);
cur = next;
} while ((cur != null) && (cur != _spareMemory) && (cur->State == EntryState.Empty));
if (size > 0) {
if (size > 0)
{
OrganizeMemory(cur, size);
size = 0;
}
}
}
/// <summary>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
/// Free allocated memory block and perform adjacent block merging
/// </summary>
/// <param name="handle">
/// Handle to the allocated memory block
/// <param name="collectPolicy">
/// </param>
public void Free(
HlsfHandle handle,
FreePolicy collectPolicy = FreePolicy.NoCollect
@@ -108,16 +110,12 @@ namespace StgSharp.HighPerformance.Memory
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
(int finalLevel, int finalSegment) = GetIndexPairOnce(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
@@ -145,16 +143,14 @@ namespace StgSharp.HighPerformance.Memory
e->State = EntryState.Empty;
// Try merge with adjacent free blocks and remove them from buckets
if (collectPolicy == FreePolicy.LazyCollect)
{
MergeAndRemoveFromBuckets(e);
}
/**/
MergeAndRemoveFromBuckets(e);
// Calculate final level and segment for the merged block
nuint finalSize = e->Size;
int finalLevel = GetLevelFromSize(finalSize);
e->Level = finalLevel;
int finalSegment = DetermineSegmentIndex(_levelSizeArray, finalSize, finalLevel);
int finalSegment = GetSegmentFromLevel(finalSize, finalLevel);
// Push the merged entry back to appropriate bucket
PushBucketToLevel(finalLevel, finalSegment, (BucketNode*)e->Position);
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Level.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.Level.cs
+15 -2
View File
@@ -62,7 +62,7 @@ namespace StgSharp.HighPerformance.Memory
level = Math.Min(level, _maxLevel);
uint levelSize = 1u << (log2Size & (~1));
int segment = ((int)(size / levelSize)) - 1;
return (level * 3) + segment;
return Math.Min((level * 3) + segment, _bucketHeads.Length - 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -82,6 +82,19 @@ namespace StgSharp.HighPerformance.Memory
return (level, segment);
}
/// <summary>
/// NOTICE: returned level may exceed maxLevel, caller is responsible for ensuring the
/// returned level is within bounds. This method is designed for performance and does not
/// perform bounds checking.
/// </summary>
/// <param name="size">
/// Size of the memory block to determine level for. This method does not assume size is
/// already aligned
/// </param>
/// <returns>
/// Level corresponding to the given size. Level is determined based on the formula: level
/// >= (log2(S) - 6) / 2
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevelFromSize(
nuint size
@@ -104,7 +117,7 @@ namespace StgSharp.HighPerformance.Memory
int level = Math.Max(0, (log2Size - 6) / 2);
// caller is responsible for ensuring level is within bounds
return Math.Min(level, _maxLevel);
return level;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.SegmentEntry.cs
+27 -13
View File
@@ -42,13 +42,12 @@ namespace StgSharp.HighPerformance.Memory
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool AssertAdjacentTo(
Entry* current,
Entry* other
Entry* front,
Entry* back
)
{
// Only a predicate; never throw here. Callers rely on false to skip merging.
return (other->Position + other->Size == current->Position) ||
(current->Position + current->Size == other->Position);
return front->Position + front->Size == back->Position;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -83,7 +82,8 @@ namespace StgSharp.HighPerformance.Memory
/// strategy.
/// </summary>
private void MergeAndRemoveFromBuckets(
Entry* entry
Entry* entry,
int maxScanRange = 4
)
{
if (entry == null) {
@@ -94,7 +94,8 @@ namespace StgSharp.HighPerformance.Memory
nuint size = entry->Size;
int originLevel = entry->Level;
int segment = GetSegmentFromLevel(size, originLevel);
do
for (int i = 0; i < maxScanRange; i++)
{
// previous direction merge
Entry* p = entry->PreviousNear;
@@ -112,9 +113,9 @@ namespace StgSharp.HighPerformance.Memory
entry->Position = p->Position;
RemoveFromBucket(originLevel, GetSegmentFromLevel(p->Size, originLevel), (BucketNode*)p->Position);
RemoveFromPositionChain(p);
} while (true);
}
do
for (int i = 0; i < maxScanRange; i++)
{
// next direction merge
Entry* n = entry->NextNear;
@@ -122,7 +123,7 @@ namespace StgSharp.HighPerformance.Memory
(n == _spareMemory) ||
(n->State != EntryState.Empty) ||
(n->Level != entry->Level) ||
!AssertAdjacentTo(n, entry))
!AssertAdjacentTo(entry, n))
{
break;
}
@@ -131,8 +132,9 @@ namespace StgSharp.HighPerformance.Memory
entry->Size += n->Size;
RemoveFromBucket(originLevel, GetSegmentFromLevel(n->Size, originLevel), (BucketNode*)n->Position);
RemoveFromPositionChain(n);
} while (true);
originLevel = int.Min(9, originLevel);
}
// originLevel = int.Min(9, originLevel);
entry->Level = GetLevelFromSize(entry->Size);
BucketNode* bucket = (BucketNode*)entry->Position;
@@ -181,13 +183,25 @@ namespace StgSharp.HighPerformance.Memory
internal struct Entry
{
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(0)] internal Entry* NextNear; // Position linked list: next pointer
[FieldOffset(8)] internal Entry* PreviousNear; // Position linked list: previous pointer
[FieldOffset(32)] internal int Level; // Level of the block in the allocator
[FieldOffset(16)] internal nuint Size; // Block size information
[FieldOffset(36)] public EntryState State;
[FieldOffset(24)] public nuint Position; // Memory block position
[FieldOffset(24)] public nuint Position; // Memory block position
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(0)] internal int NextLevel;
[FieldOffset(4)] internal int NextNear;
[FieldOffset(8)] internal int PreviousLevel;
[FieldOffset(12)] internal int PreviousNear;
[FieldOffset(16)] internal nuint Size;
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs → StgSharp.Common/HighPerformance/Memory/old/HLSFAllocator.cs
+6
View File
@@ -38,6 +38,7 @@ namespace StgSharp.HighPerformance.Memory
// Bucket system field - directly in main allocator class
private readonly BucketNode*[] _bucketHeads;
private Metadata* _metadata;
// _spareMemory is also the beginning position of allocator
private Entry* _spareMemory;
@@ -46,9 +47,12 @@ namespace StgSharp.HighPerformance.Memory
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly int _maxAllocCount;
private readonly int _maxLevel;
private int unusedMetaData;
private readonly nuint _maxAllocSize;
private readonly nuint _size;
private Ptr64 _basePtr;
private readonly SlabAllocator<Entry> _entries;
internal HybridLayerSegregatedFitAllocator(
@@ -78,6 +82,8 @@ namespace StgSharp.HighPerformance.Memory
_entries = entryManager ??
SlabAllocator.Create<Entry>(64, SlabBufferLayout.Chunked, false);
m_Buffer = (byte*)externalPointer;
_basePtr = (Ptr64)(nuint)externalPointer;
_metadata = (Metadata*)((nuint)externalPointer + byteSize);
_align = align;
_size = byteSize;
StgSharp.Common/HighPerformance/ProcessorAbstraction/M64.cs
+15
View File
@@ -71,6 +71,13 @@ namespace StgSharp.HighPerformance.ProcessorAbstraction
return value.GetHashCode();
}
public ulong GetLowerBits(
int bitCount
)
{
return value & ((1UL << bitCount) - 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T Member<T>(
int index
@@ -79,6 +86,14 @@ namespace StgSharp.HighPerformance.ProcessorAbstraction
return ref Unsafe.As<byte, T>(ref buffer[index * sizeof(T)]);
}
public void SetLowerBits(
int bitCount,
ulong bits
)
{
value = (value & (~(1UL << bitCount))) | (bits & (1UL << bitCount));
}
public static bool operator !=(
M64 left,
M64 right