youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hlsf wrong slicing solving

Browse Source
This commit is contained in:
Youmu Liu
2025-11-26 20:52:12 +08:00
parent 23914f05ac
commit 83537c7be7
9 changed files with 108 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/HLSFAllocationHandle.cs
+9 -3
View File
@@ -47,12 +47,18 @@ namespace StgSharp.HighPerformance.Memory
EntryHandle = handle;
Pointer = (byte*)handle->Position;
AllocSize = s;
// Console.WriteLine($"{(ulong)handle->PreviousNear}<->{(ulong)handle->NextNear},\t{handle->Size}");
}
public readonly ref byte this[int index]
public readonly ref byte this[uint index]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => ref Pointer[index];
get
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual<uint>(index, AllocSize);
return ref Pointer[index];
}
}
public readonly byte* Pointer { get; init; }
@@ -73,7 +79,7 @@ namespace StgSharp.HighPerformance.Memory
private HybridLayerSegregatedFitAllocationHandle _handle;
private int _index = -1;
private uint _index = ~0u;
internal Enumerator(HybridLayerSegregatedFitAllocationHandle span)
{
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.Alloc.cs
+10 -16
View File
@@ -25,18 +25,8 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using Microsoft.Win32;
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using hlsfHandle = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocationHandle;
@@ -121,13 +111,15 @@ namespace StgSharp.HighPerformance.Memory
current->NextNear = newEntry;
next->PreviousNear = newEntry;
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// set new entry
// Entry* old = current;
current = newEntry;
newEntry->Level = i;
PushLevel(i, Math.Min(blockCount - 1, 2), bucket);
// Console.WriteLine($"{(ulong)current->PreviousNear}<->{(ulong)current->NextNear},\t{current->Size}");
finalOffset += (ulong)s;
remainSize -= (ulong)s;
if (remainSize <= (ulong)levelSize)
@@ -212,6 +204,8 @@ namespace StgSharp.HighPerformance.Memory
// Entry* old = current;
current = newEntry;
// Console.WriteLine($"{(ulong)current->PreviousNear}<->{(ulong)current->NextNear},\t{current->Size}");
// update lock and data
finalOffset += (ulong)s;
remainSize -= (ulong)s;
@@ -225,6 +219,8 @@ namespace StgSharp.HighPerformance.Memory
{
e->Size = (uint)(offset);
e->Level = GetLevelFromSize((uint)offset);
// Console.WriteLine("Alloc ok");
}
}
@@ -241,7 +237,6 @@ namespace StgSharp.HighPerformance.Memory
{
nuint pos = _spareMemory->Position;
nuint blockSize = (nuint)levelSizeArray[^1] * 2; // Use last level size (16MB)
nuint newPos = pos + blockSize;
Entry* previous;
previous = _spareMemory->PreviousNear;
@@ -250,6 +245,7 @@ namespace StgSharp.HighPerformance.Memory
handle = null;
return false; // No spare memory available
}
nuint newPos = pos + blockSize;
// Check if we have enough space
if (newPos > ((nuint)m_Buffer) + _size)
@@ -270,8 +266,6 @@ namespace StgSharp.HighPerformance.Memory
BucketNode* b = (BucketNode*)handle->Position;
b->EntryRef = handle;
// PushLevel(handle->Level, DetermineSegmentIndex((uint)blockSize, handle->Level), b);
// update position linked list
handle->NextNear = _spareMemory;
handle->PreviousNear = previous;
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.BucketNode.cs
+2 -1
View File
@@ -26,6 +26,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Reflection.Metadata;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@@ -155,7 +156,7 @@ namespace StgSharp.HighPerformance.Memory
node->NextLevel = null;
node->PreviousLevel = null;
node->IsInBucket = false;
node->EntryRef = null;
return true;
}
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.SegmentEntry.cs
+19 -26
View File
@@ -108,7 +108,15 @@ namespace StgSharp.HighPerformance.Memory
/// </summary>
private void MergeAndRemoveFromBuckets(Entry* entry)
{
if (entry == null) {
// TODO
/*
* 1.有限回收，最多向上收集2层（算上当前总共3层）
* 2.单层内就可以通过前后链表遍历解决，很容易做到
* 3.由于SLAB分配器内部包含快速复用路径，因而可以考虑直接进行Free掉被合并的Entry。复用成本很低（因为SLAB既没有Version也没有IEnumerable，所以不会有额外的操作计数损耗）
* 4.后续提供HLSF.Collect全量整理API，从头遍历邻接块进行彻底的合并
*/
if (entry == null)
{
return;
}
@@ -130,14 +138,16 @@ namespace StgSharp.HighPerformance.Memory
// compute offset of current block and base position
ulong currentOffset = entry->Position - (ulong)m_Buffer;
if (prev == null ||
prev == _spareMemory ||
!AssertBackwardBoundary(currentOffset, prev, originLevel) ||
!IsAdjacentTo(entry, prev) ||
entry->Level != prev->Level ||
!TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
if (prev != null &&
prev != _spareMemory &&
AssertBackwardBoundary(currentOffset, prev, originLevel) &&
IsAdjacentTo(entry, prev) &&
TryRemoveFromBucket(originLevel, DetermineSegmentIndex(prev->Size, originLevel), (BucketNode*)prev->Position))
{
break;
// Merge
entry->Position = prev->Position;
entry->Size += prev->Size;
RemoveFromPositionChain(prev);
}
// Update linked list
@@ -148,11 +158,6 @@ namespace StgSharp.HighPerformance.Memory
}
entry->PreviousNear = prevPrev;
/**/
// Merge
entry->Position = prev->Position;
entry->Size += prev->Size;
RemoveFromPositionChain(prev);
}
while (true)
@@ -189,19 +194,6 @@ namespace StgSharp.HighPerformance.Memory
} while (originLevel < entry->Level);
}
private void RecycleEntry(Entry* entry)
{
if (entry is null) {
return;
}
entry->PreviousNear = null;
entry->NextNear = null;
// Console.WriteLine("entry recycled.");
_entries.Free((nuint)entry);
}
/// <summary>
/// Remove node from position linked list
/// </summary>
@@ -246,6 +238,7 @@ namespace StgSharp.HighPerformance.Memory
internal enum EntryState : uint
{
Spare,
Empty,
Allocated
StgSharp.Common/HighPerformance/Memory/HLSFAllocator.cs
+1 -1
View File
@@ -59,7 +59,7 @@ namespace StgSharp.HighPerformance.Memory
_bucketHeads = new BucketNode*[levelSizeArray.Length * 3 + 1];
_spareMemory = (Entry*)_entries.Allocate();
_spareMemory->State = EntryState.Empty;
_spareMemory->State = EntryState.Spare;
_spareMemory->PreviousNear = null;
_spareMemory->NextNear = null;
_spareMemory->Position = (nuint)m_Buffer;
StgSharp.Common/Internal/Graphic/gl.cs
+16 -5
View File
@@ -68,21 +68,32 @@ namespace StgSharp.Internal
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
internal static partial void InternalInitGL(int majorVersion, int minorVersion);
[LibraryImport(Native.LibName, EntryPoint = "loadGlfuncDefault", StringMarshalling = StringMarshalling.Utf8)]
[LibraryImport(
Native.LibName,
EntryPoint = "loadGlfuncDefault",
StringMarshalling = StringMarshalling.Utf8)]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
internal static partial IntPtr InternalLoadGlfuncDefault(string name);
internal static partial IntPtr InternalLoadGlFuncDefault(string name);
[LibraryImport(Native.LibName, EntryPoint = "linkShaderProgram")]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
internal static unsafe partial uint InternalLinkShaderProgram(OpenglContext* context, uint shaderProgram);
internal static unsafe partial uint InternalLinkShaderProgram(
OpenglContext* context,
uint shaderProgram);
[LibraryImport(Native.LibName, EntryPoint = "readLog")]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
internal static unsafe partial IntPtr InternalReadSSCLog();
[LibraryImport(Native.LibName, EntryPoint = "loadImageData", StringMarshalling = StringMarshalling.Utf8)]
[LibraryImport(
Native.LibName,
EntryPoint = "loadImageData",
StringMarshalling = StringMarshalling.Utf8)]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
internal static unsafe partial void InternalLoadImage(string fileName, ImageInfo* output, ImageLoader loader);
internal static unsafe partial void InternalLoadImage(
string fileName,
ImageInfo* output,
ImageLoader loader);
[LibraryImport(Native.LibName, EntryPoint = "unloadImageData")]
[UnmanagedCallConv(CallConvs =[ typeof(CallConvCdecl) ])]
StgSharpDebug/HlsfStressTest.cs
+2 -1
View File
@@ -42,7 +42,7 @@ namespace StgSharp
int.TryParse(mbStr, out int mb) &&
mb >= 64 ?
(ulong)mb * 1024UL * 1024UL :
512UL * 1024UL * 1024UL; // lower default to 512MB
1024UL * 1024UL * 1024UL; // lower default to 512MB
private static readonly uint[] SizeSweep = {
64, 128, 192, 256, 512, 768, 1024, 2048, 3072, 4096, 8192, 12288, 16384, 32768, 49152, 65536, 131072, 196608, 262144, 524288, 786432, 1_048_576, 2_097_152, 3_145_728, 4_194_304, 8_388_608, 12_582_912, 16_777_216, 25_165_824, 33_554_432
@@ -156,6 +156,7 @@ namespace StgSharp
}
}
list.Add(h);
Console.WriteLine(i);
}
sw.Stop();
double allocOpsPerSec = ops / sw.Elapsed.TotalSeconds;
StgSharpDebug/Program.cs
+2 -1
View File
@@ -45,8 +45,9 @@ namespace StgSharp
.InitializeModule();
// TestRender();
TestHlsf();
HlsfStressTest.Run();
// HlsfStressTest.Run();
/*
HybridLayerSegregatedFitAllocator hlsf = new HybridLayerSegregatedFitAllocator((nuint)(64 * (ulong)10000 * 10000));
StgSharpDebug/hlsf-test.cs
+47 -118
View File
@@ -1,30 +1,27 @@
//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="hlsf-test.cs"
// file="hlsf-test"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the “Software”), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED “AS IS”,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
@@ -46,106 +43,38 @@ namespace StgSharp
// HLSF test arena
Console.WriteLine(
"=== HLSF (Hybrid Layer Segregated Fit Allocator) Single-threaded Stress Test ===\n");
HybridLayerSegregatedFitAllocator hlsfAllocator = new HybridLayerSegregatedFitAllocator(1024 * 1024 * 1024);
// === 简洁交替测试：100次alloc与100次free交替进行，直到触发异常或达到最大循环 ===
Console.WriteLine("\n=== 简洁交替测试：100次alloc + 100次free 交替 ===");
// 使用适中的内存池大小
HybridLayerSegregatedFitAllocator hlsfAllocator = new HybridLayerSegregatedFitAllocator(16 * 1024 * 1024);
Console.WriteLine("开始单线程压力测试...\n");
// 单线程压力测试：适度的分配和释放操作
int totalCycles = 5;
int allocationsPerCycle = 1024 * 64; // 65536 次分配
for (int cycle = 0; cycle < totalCycles; cycle++)
{
Console.WriteLine($"\n=== 第 {cycle + 1} 轮压力测试 ===");
// 预生成随机数
uint[] randomSizes = new uint[allocationsPerCycle];
Random random = new Random(cycle);
for (int i = 0; i < allocationsPerCycle; i++) {
randomSizes[i] = (uint)(64 + random.Next(1024 - 64 + 1));
}
// 分配阶段
List<HybridLayerSegregatedFitAllocationHandle> handles = new List<HybridLayerSegregatedFitAllocationHandle>(allocationsPerCycle);
Stopwatch stopwatch = Stopwatch.StartNew();
for (int i = 0; i < allocationsPerCycle; i++) {
handles.Add(hlsfAllocator.Alloc(randomSizes[i]));
}
stopwatch.Stop();
Console.WriteLine($"分配完成，总计分配 {handles.Count} 个块");
Console.WriteLine($"分配耗时: {stopwatch.ElapsedMilliseconds} ms");
if (stopwatch.ElapsedMilliseconds > 0) {
Console.WriteLine($"分配速度: {allocationsPerCycle * 1000 / stopwatch.ElapsedMilliseconds} allocs/sec");
}
// 随机释放阶段
stopwatch.Restart();
for (int i = handles.Count - 1; i >= 0; i--) {
hlsfAllocator.Free(handles[i]);
}
stopwatch.Stop();
Console.WriteLine($"释放完成，总计释放 {handles.Count} 个块");
Console.WriteLine($"释放耗时: {stopwatch.ElapsedMilliseconds} ms");
if (stopwatch.ElapsedMilliseconds > 0) {
Console.WriteLine($"释放速度: {handles.Count * 1000 / stopwatch.ElapsedMilliseconds} frees/sec");
}
handles.Clear();
}
Console.WriteLine("\n=== 边界测试 ===");
// 测试最小分配
Console.WriteLine("测试最小分配 (64字节)...");
List<HybridLayerSegregatedFitAllocationHandle> minHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
for (int i = 0; i < 1000; i++) {
minHandles.Add(hlsfAllocator.Alloc(64));
}
foreach (HybridLayerSegregatedFitAllocationHandle handle in minHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("最小分配测试完成");
// 测试最大分配
Console.WriteLine("测试最大分配 (1024字节)...");
List<HybridLayerSegregatedFitAllocationHandle> maxHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
for (int i = 0; i < 1000; i++) {
maxHandles.Add(hlsfAllocator.Alloc(1024));
}
foreach (HybridLayerSegregatedFitAllocationHandle handle in maxHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("最大分配测试完成");
// 测试混合大小分配
Console.WriteLine("测试混合大小分配...");
List<HybridLayerSegregatedFitAllocationHandle> mixedHandles = new List<HybridLayerSegregatedFitAllocationHandle>();
uint[] testSizes = {
64, 128, 256, 512, 1024
int maxRounds = 10000; // 可调整
uint[] sizes = {
64u, 128u, 256u, 512u, 1024u
};
for (int i = 0; i < 2000; i++)
Random rnd = new Random(42);
for (int round = 1; round <= maxRounds; round++)
{
uint size = testSizes[i % testSizes.Length];
mixedHandles.Add(hlsfAllocator.Alloc(size));
}
foreach (HybridLayerSegregatedFitAllocationHandle handle in mixedHandles) {
hlsfAllocator.Free(handle);
}
Console.WriteLine("混合大小分配测试完成");
// 100 次分配
List<HybridLayerSegregatedFitAllocationHandle> batch = new List<HybridLayerSegregatedFitAllocationHandle>(100);
for (int i = 0; i < 100; i++)
{
uint sz = sizes[rnd.Next(sizes.Length)];
batch.Add(hlsfAllocator.Alloc(sz));
Console.WriteLine($"alloced:{i}");
}
Console.WriteLine("\n所有测试完成！HLSF分配器表现良好。");
Console.WriteLine("按任意键退出...");
Console.ReadKey();
// 100 次释放（逆序）
for (int i = batch.Count - 1; i >= 0; i--)
{
hlsfAllocator.Free(batch[i]);
Console.WriteLine($"freed:{i}");
}
if (round % 100 == 0) {
Console.WriteLine($"[交替测试] 已完成 {round} 轮 100alloc+100free");
}
}
Console.WriteLine("[交替测试] 正常结束，无异常触发");
}
}