youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tlsf briefly ok, debugging

Browse Source
This commit is contained in:
Youmu Liu
2026-05-12 18:25:48 +08:00
parent a99b19fa3f
commit bf9879f59f
5 changed files with 388 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+1 -1
View File
@@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory
}
// origin :CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
CacheLineHead* entry = (CacheLineHead*)handle.Pointer;
CacheLineHead* entry = (CacheLineHead*)handle.Address;
index = entry->Id;
return true;
}
StgSharp.Common/HighPerformance/Memory/TwoLayerSegregatedFitAllocator.cs
+363 -157
View File
@@ -50,6 +50,7 @@ namespace StgSharp.HighPerformance.Memory
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly int _maxLevel;
private int _metadataEnd;
private readonly Ptr64 _basePtr;
@@ -77,15 +78,17 @@ namespace StgSharp.HighPerformance.Memory
_basePtr = (Ptr64)(ulong)externalPointer;
_align = align;
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
_bucket = new int[GetLevel(_maxAllocSize) + 1];
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new int[GetLevel(_maxAllocSize) + 2];
_spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
_freeHandle = freeHandle ?? (static_ => { });
_metadataEnd = 0;
// init spare memory node
_spareMemory->Position = 0UL;
_spareMemory->Size = _totalSize - Metadata.SizeOf;
_spareMemory->Position = _basePtr;
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0;
@@ -120,6 +123,242 @@ namespace StgSharp.HighPerformance.Memory
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
{
if (disposing) {
_freeHandle((nuint)_basePtr);
}
disposedValue = true;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(
ulong size
)
{
int origin = 57 - BitOperations.LeadingZeroCount(size);
return int.Min(origin, _maxLevel);
}
/// <summary>
/// Put a metadata node back to free list and waiting for next allocation.
/// </summary>
/// <param name="node">
///
/// </param>
private void RecycleNode(
Metadata* node
)
{
int nextIdx = _spareMemory->NextLevel;
int nodeIdx = (int)(_spareMemory - node);
int level = GetLevel(node->Size);
Metadata* next = &_spareMemory[-nextIdx];
int nextLevelIdx = node->NextLevel;
int prevLevelIdx = node->PreviousLevel;
if (nextIdx == 0)
{
_spareMemory->NextLevel = nodeIdx;
node->NextLevel = 0;
return;
} else
{
// add to free list
next->PreviousLevel = nodeIdx;
_spareMemory->NextLevel = nodeIdx;
node->NextLevel = nextIdx;
node->PreviousLevel = 0;
}
// remove from position chain
int prevNearIdx = node->PreviousNear;
int nextNearIdx = node->NextNear;
Metadata* prevNear = &_spareMemory[-prevNearIdx];
Metadata* nextNear = &_spareMemory[-nextNearIdx];
prevNear->NextNear = nextNearIdx;
nextNear->PreviousNear = prevNearIdx;
node->PreviousNear = 0;
node->NextNear = 0;
node->Position = 0;
node->SizeForSpare = 0;
// remove from level bucket
if (_bucket[level] == nodeIdx)
{
_bucket[level] = nextLevelIdx;
if (nextLevelIdx != 0)
{
Metadata* nextLevel = &_spareMemory[-nextLevelIdx];
nextLevel->PreviousLevel = 0;
}
} else
{
if (nextLevelIdx != 0)
{
Metadata* nextLevel = &_spareMemory[-nextLevelIdx];
nextLevel->PreviousLevel = prevLevelIdx;
}
if (prevLevelIdx != 0)
{
Metadata* prevLevel = &_spareMemory[-prevLevelIdx];
prevLevel->NextLevel = nextLevelIdx;
}
node->PreviousLevel = 0;
node->NextLevel = 0;
}
}
private bool TryAllocNewChunk(
ulong size,
out Metadata* meta
)
{
int idx = _spareMemory->NextLevel;
if (idx != 0)
{
meta = &_spareMemory[-idx];
_spareMemory->NextLevel = meta->NextLevel;
} else
{
_metadataEnd++;
meta = &_spareMemory[-_metadataEnd];
// TODO: correctly set position and size for new chunk
}
ulong spareSize = _spareMemory->Size;
if (spareSize < Metadata.SizeOf + size)
{
meta = null;
return false;
}
spareSize -= Metadata.SizeOf + size;
_spareMemory->Size = spareSize;
_spareMemory->Position += size;
meta->Size = size;
// insert the new chunk to position chain
int nearIdx = _spareMemory->PreviousNear;
Metadata* near = &_spareMemory[-nearIdx];
near->NextNear = _metadataEnd;
meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = _metadataEnd;
meta->NextNear = 0;
return true;
}
private bool TryGetNewMetadata(
out Metadata* meta
)
{
ulong spareSize = _spareMemory->Size;
if (spareSize < Metadata.SizeOf)
{
meta = null;
return false;
}
_metadataEnd++;
meta = &_spareMemory[-_metadataEnd];
spareSize -= Metadata.SizeOf;
_spareMemory->Size = spareSize;
return true;
}
[StructLayout(LayoutKind.Explicit)]
public readonly struct Handle
{
[FieldOffset(0)] internal readonly Metadata* _meta;
[FieldOffset(8)] public readonly void* Address;
[FieldOffset(16)] public readonly Ptr64 Size;
internal Handle(
Metadata* meta,
Ptr64 baseAddress
)
{
_meta = meta;
Address = (void*)(ulong)meta->Position;
Size = meta->Size;
}
[Obsolete("You should not create instance from default. Alloc from allocator please")]
public Handle() { }
public ref byte this[
int offset
] => ref ((byte*)Address)[offset];
public Span<byte> AsSpan(
int range = -1
)
{
return new Span<byte>(Address, range < 0 ? (int)(ulong)Size : range);
}
}
internal enum AllocationState : byte
{
SpareOrCollect = 0,
Free = 1,
Alloc = 2,
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(24)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(30)] internal AllocationState State;
[FieldOffset(31)] internal sbyte Level;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 Size
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
}
#region alloc and free
public Handle Alloc(
nuint size
)
@@ -176,15 +415,37 @@ namespace StgSharp.HighPerformance.Memory
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
return new Handle(entry, _basePtr);
}
public void Collect() { }
public void Dispose()
public void Collect()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
int currentIdx = _spareMemory->NextNear;
do
{
if (currentIdx == 0)
{
break;
}
Metadata* current = &_spareMemory[-currentIdx];
if (current->State == AllocationState.Free)
{
Ptr64 size = 0;
int nextIdx = current->NextNear;
Metadata* next = &_spareMemory[-nextIdx];
while (nextIdx != 0 && next->State != AllocationState.Free)
{
size += next->Size;
RecycleNode(next);
nextIdx = next->NextNear;
}
if (size > 0) {
current->Size += size;
}
}
currentIdx = current->NextNear;
} while (true);
}
public void Free(
@@ -193,14 +454,55 @@ namespace StgSharp.HighPerformance.Memory
)
{
Metadata* head = handle._meta;
int headIdx = (int)(_spareMemory - head);
Ptr64 size = handle.Size;
for (int i = 0; i < collectScanWindow; i++)
{
// forward scan
// forward scan, previous direction
int forwardIdx = head->PreviousNear;
if (forwardIdx == 0)
{
break;
}
Metadata* prev = &_spareMemory[-forwardIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->Size;
head->Position = prev->Position;
RecycleNode(prev);
}
for (int i = 0; i < collectScanWindow; i++)
{
// backward scan
// backward scan, next direction
int nextIdx = head->NextNear;
if (nextIdx == 0)
{
break;
}
Metadata* prev = &_spareMemory[-nextIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->Size;
RecycleNode(prev);
}
// recycle to level;
head->Size = size;
head->State = AllocationState.Free;
int level = GetLevel(head->Size);
int bucketHeadIdx = _bucket[level];
if (bucketHeadIdx != 0)
{
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
bucketHead->PreviousLevel = headIdx;
head->NextLevel = bucketHeadIdx;
}
_bucket[level] = headIdx;
return;
}
public bool TryAlloc(
@@ -208,162 +510,66 @@ namespace StgSharp.HighPerformance.Memory
out Handle h
)
{
throw new NotImplementedException();
}
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
if (allocSize > _maxAllocSize)
{
if (disposing) {
_freeHandle((nuint)_basePtr);
h = default;
return false;
}
int level = GetLevel(allocSize);
Metadata* entry = null;
for (int i = level; i < _bucket.Length; i++)
{
int idx = _bucket[i];
if (idx != 0)
{
entry = &_spareMemory[-idx];
_bucket[i] = entry->NextLevel;
break;
}
disposedValue = true;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int GetLevel(
ulong size
)
{
return 57 - BitOperations.LeadingZeroCount(size);
}
private bool TryAllocNewChunk(
ulong size,
out Metadata* meta
)
{
int idx = _spareMemory->NextLevel;
if (idx != 0)
if (entry is null && !TryAllocNewChunk(allocSize, out entry))
{
meta = &_spareMemory[-idx];
_spareMemory->NextLevel = meta->NextLevel;
return true;
}
ulong spareSize = _spareMemory->Size;
if (spareSize < Metadata.SizeOf + size)
{
meta = null;
h = default;
return false;
}
_metadataEnd++;
meta = &_spareMemory[-_metadataEnd];
spareSize -= Metadata.SizeOf + size;
_spareMemory->Size = spareSize;
_spareMemory->Position += size;
meta->Size = size;
int entryIdx = (int)(_spareMemory - entry);
ulong entrySize = entry->Size;
ulong remain = entrySize - allocSize;
// insert the new chunk to position chain
int nearIdx = _spareMemory->PreviousNear;
Metadata* near = &_spareMemory[-nearIdx];
near->NextNear = _metadataEnd;
meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = _metadataEnd;
meta->NextNear = 0;
// slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
{
int newEntryIdx = (int)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize;
newEntry->Size = remain;
int newLevel = GetLevel(remain);
int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
newEntry->NextLevel = bucketHeadIdx;
bucketHead->PreviousLevel = newEntryIdx;
} else
{
newEntry->NextLevel = 0;
}
newEntry->PreviousLevel = 0;
_bucket[newLevel] = newEntryIdx;
// push to position chain
int afterIdx = entry->NextNear;
Metadata* after = &_spareMemory[-afterIdx];
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
h = new Handle(entry, _basePtr);
return true;
}
private bool TryGetNewMetadata(
out Metadata* meta
)
{
ulong spareSize = _spareMemory->Size;
if (spareSize < Metadata.SizeOf)
{
meta = null;
return false;
}
_metadataEnd++;
meta = &_spareMemory[-_metadataEnd];
spareSize -= Metadata.SizeOf;
_spareMemory->Size = spareSize;
return true;
}
[StructLayout(LayoutKind.Explicit)]
public readonly struct Handle
{
[FieldOffset(0)]internal readonly Metadata* _meta;
[FieldOffset(8)]public readonly void* Pointer;
[FieldOffset(16)]public readonly Ptr64 Size;
internal Handle(
Metadata* meta,
Ptr64 baseAddress
)
{
_meta = meta;
Pointer = (void*)(ulong)(meta->Position + baseAddress);
Size = meta->Size;
}
[Obsolete("You should not create instance from default. Alloc from allocator please")]
public Handle() { }
public ref byte this[
int offset
] => ref ((byte*)Pointer)[offset];
public Span<byte> AsSpan(
int range = -1
)
{
return new Span<byte>(Pointer, range < 0 ? (int)(ulong)Size : range);
}
}
internal enum AllocationState : short
{
Free = 0,
Used = 1,
Split = 2,
Merged = 3
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(24)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(30)] internal AllocationState State;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 Size
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
}
#endregion
}
}
StgSharp.Mathematics/Numeric/Linear/MatrixStorage.cs
+1 -1
View File
@@ -62,7 +62,7 @@ namespace StgSharp.Mathematics.Numeric
internal readonly Tlsf _allocator = allocator;
protected internal override unsafe T* BufferPointer => (T*)_handle.Pointer;
protected internal override unsafe T* BufferPointer => (T*)_handle.Address;
public override void Dispose()
{
StgSharpDebug/Program.cs
+2 -2
View File
@@ -53,9 +53,9 @@ namespace StgSharp
.InitializeModule();
// TestRegexAnalyse.Run();
TestHlsf();
// TestHlsf();
// HlsfStressTest.Run();
TlsfStressTest.Run();
// TestSwiss();
// TestL4Prefetch();
StgSharpDebug/hlsf-test.cs
+21 -7
View File
@@ -85,10 +85,17 @@ namespace StgSharp
List<Tlsf.Handle> batch = new(maxCount);
for (int i = 0; i < maxCount; i++)
{
uint sz = sizes[rnd.Next(sizes.Length)];
batch.Add(hlsfAllocator.Alloc(sz));
// Console.WriteLine($"alloced:{i}");
try
{
uint sz = sizes[rnd.Next(sizes.Length)];
batch.Add(hlsfAllocator.Alloc(sz));
}
catch (Exception e)
{
Console.WriteLine($"alloced:{round},{i}");
Console.WriteLine(e);
throw;
}
}
for (int i = 5; i < maxCount; i += 10)
{
@@ -99,9 +106,16 @@ namespace StgSharp
}
for (int i = batch.Count - 1; i >= 0; i--)
{
hlsfAllocator.Free(batch[i]);
// Console.WriteLine($"freed:{i}");
try
{
hlsfAllocator.Free(batch[i]);
}
catch (Exception e)
{
Console.WriteLine($"freed:{round},{i}");
Console.WriteLine(e);
throw;
}
}
if (round % 100 == 0)
{