youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compress L4 metadata into 32 bytes

Browse Source
This commit is contained in:
Youmu Liu
2026-06-02 18:27:58 +08:00
parent 07f8178431
commit 9f3fc3b345
7 changed files with 156 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+26 -47
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.CacheLine"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -28,8 +28,6 @@
using StgSharp.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices;
using Tlsf = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
@@ -48,71 +46,52 @@ namespace StgSharp.HighPerformance.Memory
private unsafe struct CacheLineMetadata
{
#region reference counting and state information
[FieldOffset(24)] internal int RefCount;
// [FieldOffset(28)] internal readonly short Id;
#endregion
#region tlsf area
// TLSF area use only 24 bytes
internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(8)] private ulong PositionMask;
[FieldOffset(16)] private ulong SizeMask;
[FieldOffset(24)] internal nuint Origin;
[FieldOffset(14)] internal AllocationState State;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
[FieldOffset(0)] public short NextLevel;
[FieldOffset(2)] public short NextNear;
[FieldOffset(4)] public short PreviousLevel;
[FieldOffset(6)] public short PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
#endregion
public Ptr64 SizeForAllocator
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
#region buffer
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
[FieldOffset(8)] public uint PositionMask;
[FieldOffset(12)] public uint SizeMask;
#endregion
#region cache line information
#region mask
[FieldOffset(34)] internal ushort Profile;
[FieldOffset(24)] internal nuint Origin;
[FieldOffset(16)] internal AllocationState State;
[FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion
}
[StructLayout(LayoutKind.Sequential)]
public unsafe ref struct Handle : IDisposable
#pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit)]
public unsafe struct Handle : IDisposable
{
private CacheLineMetadata* _head;
private nuint _handle;
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
[FieldOffset(0)] private nuint _handle;
private Handle(nuint handle, CacheLineMetadata* cacheRefCount)
private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{
_handle = handle;
_head = cacheRefCount;
@@ -133,6 +112,6 @@ namespace StgSharp.HighPerformance.Memory
}
}
#pragma warning restore IDE0032
}
}
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+9 -17
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -25,18 +25,6 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
using Tlsf = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
@@ -52,7 +40,9 @@ namespace StgSharp.HighPerformance.Memory
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(int count = 1)
private unsafe int Evict(
int count = 1
)
{
count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount;
@@ -71,7 +61,9 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(int index)
private unsafe bool TryEvictAt(
int index
)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
@@ -79,7 +71,7 @@ namespace StgSharp.HighPerformance.Memory
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs
-6
View File
@@ -25,12 +25,6 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public sealed class L4PredictUnregisteredException : Exception
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+4 -4
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -105,14 +105,14 @@ namespace StgSharp.HighPerformance.Memory
}
head = _head + index;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs
+90 -54
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4.Tlsf"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -53,15 +53,19 @@ namespace StgSharp.HighPerformance.Memory
private int _metadataEnd;
private readonly Ptr64 _basePtr;
private readonly ulong _maxAllocSize;
private readonly ulong _totalSize;
private readonly uint _maxAllocSize;
private readonly uint _totalSize;
internal TLSF(nuint byteSize, ulong maxBlockSize, void* externalPointer)
internal TLSF(
uint byteSize,
uint maxBlockSize,
void* externalPointer
)
{
_totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer;
_align = 4096; // align of a memory page
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits)));
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2];
@@ -70,8 +74,8 @@ namespace StgSharp.HighPerformance.Memory
// init spare memory node
_spareMemory->Position = _basePtr;
_spareMemory->SizeForSpare = _totalSize - CacheLineMetadata.SizeOf;
_spareMemory->PositionMask = 0;
_spareMemory->SizeMask = _totalSize;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0;
@@ -79,9 +83,11 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(ulong size)
private int GetLevel(
uint size
)
{
int origin = 57 - BitOperations.LeadingZeroCount(size);
int origin = 25 - BitOperations.LeadingZeroCount(size);
return int.Min(origin, _maxLevel);
}
@@ -91,10 +97,12 @@ namespace StgSharp.HighPerformance.Memory
/// <param name="node">
///
/// </param>
private void RecycleNode(CacheLineMetadata* node)
private void RecycleNode(
CacheLineMetadata* node
)
{
short nodeIdx = (short)(_spareMemory - node);
int level = GetLevel(node->SizeForAllocator);
int level = GetLevel(node->SizeMask);
short nextLevelIdx = node->NextLevel;
short prevLevelIdx = node->PreviousLevel;
RemoveFromBucket(level, node, nodeIdx);
@@ -115,8 +123,8 @@ namespace StgSharp.HighPerformance.Memory
node->PreviousNear = 0;
node->NextNear = 0;
node->Position = 0;
node->SizeForSpare = 0;
node->PositionMask = 0;
node->SizeMask = 0;
node->State = AllocationState.SpareOrCollect;
node->PreviousLevel = 0;
@@ -131,12 +139,19 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(CacheLineMetadata* node, int nodeIdx)
private void RemoveFromBucket(
CacheLineMetadata* node,
int nodeIdx
)
{
RemoveFromBucket(GetLevel(node->SizeForAllocator), node, nodeIdx);
RemoveFromBucket(GetLevel(node->SizeMask), node, nodeIdx);
}
private void RemoveFromBucket(int level, CacheLineMetadata* node, int nodeIdx)
private void RemoveFromBucket(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{
int currentLevel = level;
short currentNextLevelIdx = node->NextLevel;
@@ -160,7 +175,11 @@ namespace StgSharp.HighPerformance.Memory
}
// Assuming node to remove is head f bucket
private void RemoveFromBucketHead(int level, CacheLineMetadata* node, int nodeIdx)
private void RemoveFromBucketHead(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{
int currentLevel = level;
short currentNextLevelIdx = node->NextLevel;
@@ -187,10 +206,13 @@ namespace StgSharp.HighPerformance.Memory
node->NextLevel = 0;
}
private bool TryAllocNewChunk(ulong size, out CacheLineMetadata* meta)
private bool TryAllocNewChunk(
uint size,
out CacheLineMetadata* meta
)
{
short idx = _spareMemory->NextLevel;
ulong spareSize = _spareMemory->SizeForAllocator;
uint spareSize = _spareMemory->SizeMask;
if (idx != 0)
{
if (spareSize < size)
@@ -211,13 +233,13 @@ namespace StgSharp.HighPerformance.Memory
return false;
}
spareSize -= size;
meta->SizeForAllocator = size;
meta->Position = _spareMemory->Position;
meta->SizeMask = size;
meta->PositionMask = _spareMemory->PositionMask;
meta->State = AllocationState.Free;
meta->PreviousLevel = 0;
meta->NextLevel = 0;
_spareMemory->SizeForAllocator = spareSize;
_spareMemory->Position += size;
_spareMemory->SizeMask = spareSize;
_spareMemory->PositionMask += size;
// insert the new chunk to position chain
short nearIdx = _spareMemory->PreviousNear;
@@ -229,9 +251,11 @@ namespace StgSharp.HighPerformance.Memory
return true;
}
private bool TryGetNewMetadata(out CacheLineMetadata* meta)
private bool TryGetNewMetadata(
out CacheLineMetadata* meta
)
{
ulong spareSize = _spareMemory->SizeForAllocator;
ulong spareSize = _spareMemory->SizeMask;
if (_metadataEnd >= MaxCacheLineCount)
{
meta = null;
@@ -242,7 +266,10 @@ namespace StgSharp.HighPerformance.Memory
return true;
}
private bool TryTakeBucketEntry(ulong minimumSize, out CacheLineMetadata* entry)
private bool TryTakeBucketEntry(
uint minimumSize,
out CacheLineMetadata* entry
)
{
int level = GetLevel(minimumSize);
@@ -252,7 +279,7 @@ namespace StgSharp.HighPerformance.Memory
while (idx != 0)
{
CacheLineMetadata* candidate = &_spareMemory[idx];
if (candidate->SizeForAllocator >= minimumSize)
if (candidate->SizeMask >= minimumSize)
{
RemoveFromBucket(level, candidate, idx);
entry = candidate;
@@ -279,9 +306,11 @@ namespace StgSharp.HighPerformance.Memory
#region alloc and free
public int Alloc(nuint size)
public int Alloc(
uint size
)
{
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
}
@@ -290,18 +319,19 @@ namespace StgSharp.HighPerformance.Memory
throw new OverflowException("No available ");
}
short entryIdx = (short)(_spareMemory - entry);
ulong entrySize = entry->SizeForAllocator;
uint entrySize = entry->SizeMask;
if (entrySize < allocSize) {
throw new OverflowException("Bucket returned an undersized block.");
}
ulong remain = entrySize - allocSize;
uint remain = entrySize - allocSize;
// slice tail
if (remain > ulong.Min(4096UL, entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
if (remain > ulong.Min(4096UL, entrySize >> 7) &&
TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize;
newEntry->SizeForAllocator = remain;
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel];
@@ -325,7 +355,7 @@ namespace StgSharp.HighPerformance.Memory
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize) {
if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
return entryIdx;
@@ -344,7 +374,7 @@ namespace StgSharp.HighPerformance.Memory
if (current->State == AllocationState.Free)
{
// scan next
Ptr64 size = 0;
uint size = 0;
int nextIdx = current->NextNear;
while (nextIdx != 0)
{
@@ -353,15 +383,15 @@ namespace StgSharp.HighPerformance.Memory
{
break;
}
size += next->SizeForAllocator;
size += next->SizeMask;
RecycleNode(next);
nextIdx = current->NextNear;
}
if (size > 0)
{
current->SizeForAllocator += size;
current->SizeMask += size;
RemoveFromBucket(current, currentIdx);
int newLevel = GetLevel(current->SizeForAllocator);
int newLevel = GetLevel(current->SizeMask);
short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
@@ -377,11 +407,14 @@ namespace StgSharp.HighPerformance.Memory
} while (true);
}
public void Free(int handle, int collectScanWindow = 4)
public void Free(
int handle,
int collectScanWindow = 4
)
{
CacheLineMetadata* head = &_spareMemory[handle];
short headIdx = (short)(_spareMemory - head);
Ptr64 size = head->SizeForAllocator;
uint size = head->SizeMask;
for (int i = 0; i < collectScanWindow; i++)
{
// forward scan, previous direction
@@ -395,8 +428,8 @@ namespace StgSharp.HighPerformance.Memory
{
break;
}
size += prev->SizeForAllocator;
head->Position = prev->Position;
size += prev->SizeMask;
head->PositionMask = prev->PositionMask;
RecycleNode(prev);
}
for (int i = 0; i < collectScanWindow; i++)
@@ -412,15 +445,15 @@ namespace StgSharp.HighPerformance.Memory
{
break;
}
size += prev->SizeForAllocator;
size += prev->SizeMask;
RecycleNode(prev);
}
// recycle to level;
head->SizeForAllocator = size;
head->SizeMask = size;
head->State = AllocationState.Free;
head->PreviousLevel = 0;
int level = GetLevel(head->SizeForAllocator);
int level = GetLevel(head->SizeMask);
short bucketHeadIdx = _bucket[level];
head->NextLevel = 0;
if (bucketHeadIdx != 0)
@@ -433,9 +466,12 @@ namespace StgSharp.HighPerformance.Memory
return;
}
public bool TryAlloc(nuint size, out int h)
public bool TryAlloc(
uint size,
out int h
)
{
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize)
{
h = -1;
@@ -448,20 +484,20 @@ namespace StgSharp.HighPerformance.Memory
return false;
}
short entryIdx = (short)(_spareMemory - entry);
ulong entrySize = entry->SizeForAllocator;
uint entrySize = entry->SizeMask;
if (entrySize < allocSize)
{
h = -1;
return false;
}
ulong remain = entrySize - allocSize;
uint remain = entrySize - allocSize;
// slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize;
newEntry->SizeForAllocator = remain;
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel];
@@ -487,7 +523,7 @@ namespace StgSharp.HighPerformance.Memory
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize)
if (entry->PositionMask - (ulong)_basePtr > _totalSize)
{
h = default;
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
StgSharp.Common/HighPerformance/Memory/L4.cs
+19 -7
View File
@@ -2,8 +2,8 @@
// -----------------------------------------------------------------------
// file="L4"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
// AuthorGroup: Nitload
// Copyright (c) Nitload. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
@@ -69,8 +69,15 @@ namespace StgSharp.HighPerformance.Memory
private readonly TLSF _bufferAllocator;
public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, nuint maxCacheLineSize = 8192)
public L4(
int bindedNuma,
uint bufferSize = 8192 * 8192,
int align = 16,
uint maxCacheLineSize = 8192
)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
@@ -116,7 +123,9 @@ namespace StgSharp.HighPerformance.Memory
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(Handle line)
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
@@ -149,7 +158,9 @@ namespace StgSharp.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(ref CacheLineDescription prediction)
public Handle Map(
ref CacheLineDescription prediction
)
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
@@ -179,8 +190,9 @@ namespace StgSharp.HighPerformance.Memory
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref (*_mapCount)[entry->Id]);
return new Handle((nuint)entry->Position, ref entry->RefCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
StgSharp.Native/sn-head/sn_intrinsic.h
+8
View File
@@ -47,6 +47,14 @@ typedef union scalar_pack {
uint64_t data[8];
} scalar_pack;
typedef struct sn_L4_cache_head {
uint64_t reverse_mask_0;
uint32_t position_mask;
uint32_t size;
uint64_t reverse_mask_1;
uint64_t reverse_mask_2;
} sn_L4_cache_head;
typedef struct matrix_task {
void *mat_0;
void *mat_1;