youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working on L4's tlsf

Browse Source
This commit is contained in:
Youmu Liu
2026-05-14 20:15:07 +08:00
parent c9fb11b47b
commit b87943495d
5 changed files with 126 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+50 -13
View File
@@ -51,12 +51,57 @@ namespace StgSharp.HighPerformance.Memory
} }
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)] internal enum AllocationState : byte
private unsafe struct CacheLineHead
{ {
[FieldOffset(0)] internal Tlsf.Handle AllocatorEntry; SpareOrCollect = 0,
[FieldOffset(0)] internal readonly M512 Buffer; Free = 1,
Alloc = 2,
}
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)]
private unsafe struct CacheLineMetadata
{
#region lsf area
internal const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
[FieldOffset(0)] private readonly M512 FillingMask;
[FieldOffset(16)] private ulong PositionMask;
[FieldOffset(24)] private ulong SizeMask;
[FieldOffset(30)] internal AllocationState State;
[FieldOffset(31)] internal sbyte Level;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 SizeForAllocator
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#endregion
#region reference counting and state information #region reference counting and state information
@@ -65,21 +110,13 @@ namespace StgSharp.HighPerformance.Memory
#endregion #endregion
#region hlsf entry alias
public readonly Ptr64 Address => AllocatorEntry.Address;
public readonly Ptr64 Size => AllocatorEntry.Size;
#endregion
#region cache line information #region cache line information
[FieldOffset(32)] internal short Predictor; [FieldOffset(32)] internal short Predictor;
[FieldOffset(34)] internal ushort Profile; [FieldOffset(34)] internal ushort Profile;
[FieldOffset(36)] internal nuint Origin; [FieldOffset(36)] internal nuint Origin;
#endregion #endregion
} }
} }
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+3 -6
View File
@@ -87,10 +87,7 @@ namespace StgSharp.HighPerformance.Memory
int id int id
) )
{ {
throw new NotImplementedException(); _bufferAllocator.Free(id);
/*
Tlsf* entry = (Tlsf*)(_entryManager._basePtr + id);
_bufferAllocator.Free(entry);
/**/ /**/
} }
@@ -100,7 +97,7 @@ namespace StgSharp.HighPerformance.Memory
Span<IL4Predict> predictorSpan Span<IL4Predict> predictorSpan
) )
{ {
CacheLineHead* head = &_head[index]; CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{ {
// EvictLine(i + index); // EvictLine(i + index);
@@ -108,7 +105,7 @@ namespace StgSharp.HighPerformance.Memory
int id = head->Predictor; int id = head->Predictor;
IL4Predict predict = predictorSpan[id] ?? IL4Predict predict = predictorSpan[id] ??
throw new L4PredictUnregisteredException(id); throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Address, (int)(ulong)head->Size)); predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)(ulong)head->Position, (int)(ulong)head->SizeForAllocator));
RecycleLine(index); RecycleLine(index);
(*_mapCount)[index] = 0; (*_mapCount)[index] = 0;
(*_predictCount)[index] = int.MaxValue; (*_predictCount)[index] = int.MaxValue;
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+3 -3
View File
@@ -74,13 +74,13 @@ namespace StgSharp.HighPerformance.Memory
// now prefetch the misses // now prefetch the misses
for (int i = 0; i < missIndex; i++) for (int i = 0; i < missIndex; i++)
{ {
CacheLineHead* head; CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i]; ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address; nuint address = pHandle.Address;
int index; int index;
if (_map.TryGet(address, out nuint handle)) if (_map.TryGet(address, out nuint handle))
{ {
head = (CacheLineHead*)handle; head = (CacheLineMetadata*)handle;
index = head->Id; index = head->Id;
(*_predictCount)[index] = 0; // reset eviction counter (*_predictCount)[index] = 0; // reset eviction counter
@@ -146,7 +146,7 @@ namespace StgSharp.HighPerformance.Memory
(*_mapCount)[index] = 1; (*_mapCount)[index] = 1;
// pre fetch them here // pre fetch them here
predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Address, (int)(ulong)head->Size)); predict.Prefetch(head->Origin, head->Profile, new Span<byte>((byte*)(ulong)head->Address, (int)(ulong)head->SizeForAllocator));
_ = _map.TryAddOrSet(address, (nuint)head, out _); _ = _map.TryAddOrSet(address, (nuint)head, out _);
} }
_ = Interlocked.Add(ref _aheadCount, actualCount); _ = Interlocked.Add(ref _aheadCount, actualCount);
StgSharp.Common/HighPerformance/Memory/L4.Tlsf.cs
+57 -103
View File
@@ -25,6 +25,7 @@
// //
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using StgSharp.HighPerformance.ProcessorAbstraction;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@@ -41,13 +42,12 @@ namespace StgSharp.HighPerformance.Memory
private unsafe class TLSF private unsafe class TLSF
{ {
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
public const int MaxCacheLineCount = 8192; public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * Metadata.SizeOf; public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
private readonly int[] _bucket; private readonly int[] _bucket;
private readonly Metadata* _spareMemory; private readonly CacheLineMetadata* _spareMemory;
private readonly int _align; private readonly int _align;
private readonly int _maxLevel; private readonly int _maxLevel;
@@ -65,17 +65,17 @@ namespace StgSharp.HighPerformance.Memory
_totalSize = byteSize; _totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer; _basePtr = (Ptr64)(ulong)externalPointer;
_align = 4096; // align of a memory page _align = 4096; // align of a memory page
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits))); _maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits)));
_maxLevel = int.MaxValue; _maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize); _maxLevel = GetLevel(_maxAllocSize);
_bucket = new int[GetLevel(_maxAllocSize) + 2]; _bucket = new int[GetLevel(_maxAllocSize) + 2];
_spareMemory = (Metadata*)(ulong)(_basePtr + _totalSize); _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0; _metadataEnd = 0;
// init spare memory node // init spare memory node
_spareMemory->Position = _basePtr; _spareMemory->Position = _basePtr;
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf; _spareMemory->SizeForSpare = _totalSize - CacheLineMetadata.SizeOf;
_spareMemory->NextLevel = 0; _spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0; _spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0; _spareMemory->PreviousLevel = 0;
@@ -98,11 +98,11 @@ namespace StgSharp.HighPerformance.Memory
/// ///
/// </param> /// </param>
private void RecycleNode( private void RecycleNode(
Metadata* node CacheLineMetadata* node
) )
{ {
int nodeIdx = (int)(_spareMemory - node); int nodeIdx = (int)(_spareMemory - node);
int level = GetLevel(node->Size); int level = GetLevel(node->SizeForAllocator);
int nextLevelIdx = node->NextLevel; int nextLevelIdx = node->NextLevel;
int prevLevelIdx = node->PreviousLevel; int prevLevelIdx = node->PreviousLevel;
RemoveFromBucket(level, node, nodeIdx); RemoveFromBucket(level, node, nodeIdx);
@@ -110,11 +110,11 @@ namespace StgSharp.HighPerformance.Memory
// remove from position chain // remove from position chain
int prevNearIdx = node->PreviousNear; int prevNearIdx = node->PreviousNear;
int nextNearIdx = node->NextNear; int nextNearIdx = node->NextNear;
Metadata* prevNear = &_spareMemory[prevNearIdx]; CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
prevNear->NextNear = nextNearIdx; prevNear->NextNear = nextNearIdx;
if (nextNearIdx != 0) if (nextNearIdx != 0)
{ {
Metadata* nextNear = &_spareMemory[nextNearIdx]; CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
nextNear->PreviousNear = prevNearIdx; nextNear->PreviousNear = prevNearIdx;
} else } else
{ {
@@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory
int nextIdx = _spareMemory->NextLevel; int nextIdx = _spareMemory->NextLevel;
if (nextIdx != 0) if (nextIdx != 0)
{ {
Metadata* next = &_spareMemory[nextIdx]; CacheLineMetadata* next = &_spareMemory[nextIdx];
next->PreviousLevel = nodeIdx; next->PreviousLevel = nodeIdx;
} }
node->NextLevel = nextIdx; node->NextLevel = nextIdx;
@@ -140,16 +140,16 @@ namespace StgSharp.HighPerformance.Memory
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket( private void RemoveFromBucket(
Metadata* node, CacheLineMetadata* node,
int nodeIdx int nodeIdx
) )
{ {
RemoveFromBucket(GetLevel(node->Size), node, nodeIdx); RemoveFromBucket(GetLevel(node->SizeForAllocator), node, nodeIdx);
} }
private void RemoveFromBucket( private void RemoveFromBucket(
int level, int level,
Metadata* node, CacheLineMetadata* node,
int nodeIdx int nodeIdx
) )
{ {
@@ -162,12 +162,12 @@ namespace StgSharp.HighPerformance.Memory
_bucket[currentLevel] = currentNextLevelIdx; _bucket[currentLevel] = currentNextLevelIdx;
} else if (currentPrevLevelIdx != 0) } else if (currentPrevLevelIdx != 0)
{ {
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx]; CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
prevLevel->NextLevel = currentNextLevelIdx; prevLevel->NextLevel = currentNextLevelIdx;
} }
if (currentNextLevelIdx != 0) if (currentNextLevelIdx != 0)
{ {
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx]; CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
nextLevel->PreviousLevel = currentPrevLevelIdx; nextLevel->PreviousLevel = currentPrevLevelIdx;
} }
node->PreviousLevel = 0; node->PreviousLevel = 0;
@@ -177,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory
// Assuming node to remove is head f bucket // Assuming node to remove is head f bucket
private void RemoveFromBucketHead( private void RemoveFromBucketHead(
int level, int level,
Metadata* node, CacheLineMetadata* node,
int nodeIdx int nodeIdx
) )
{ {
@@ -199,7 +199,7 @@ namespace StgSharp.HighPerformance.Memory
/**/ /**/
if (currentNextLevelIdx != 0) if (currentNextLevelIdx != 0)
{ {
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx]; CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
nextLevel->PreviousLevel = 0; nextLevel->PreviousLevel = 0;
} }
node->PreviousLevel = 0; node->PreviousLevel = 0;
@@ -208,11 +208,11 @@ namespace StgSharp.HighPerformance.Memory
private bool TryAllocNewChunk( private bool TryAllocNewChunk(
ulong size, ulong size,
out Metadata* meta out CacheLineMetadata* meta
) )
{ {
int idx = _spareMemory->NextLevel; int idx = _spareMemory->NextLevel;
ulong spareSize = _spareMemory->Size; ulong spareSize = _spareMemory->SizeForAllocator;
if (idx != 0) if (idx != 0)
{ {
if (spareSize < size) if (spareSize < size)
@@ -224,7 +224,7 @@ namespace StgSharp.HighPerformance.Memory
int nextSpareIdx = meta->NextLevel; int nextSpareIdx = meta->NextLevel;
if (nextSpareIdx != 0) if (nextSpareIdx != 0)
{ {
Metadata* nextSpare = &_spareMemory[nextSpareIdx]; CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
nextSpare->PreviousLevel = 0; nextSpare->PreviousLevel = 0;
} }
_spareMemory->NextLevel = nextSpareIdx; _spareMemory->NextLevel = nextSpareIdx;
@@ -233,17 +233,17 @@ namespace StgSharp.HighPerformance.Memory
return false; return false;
} }
spareSize -= size; spareSize -= size;
meta->Size = size; meta->SizeForAllocator = size;
meta->Position = _spareMemory->Position; meta->Position = _spareMemory->Position;
meta->State = AllocationState.Free; meta->State = AllocationState.Free;
meta->PreviousLevel = 0; meta->PreviousLevel = 0;
meta->NextLevel = 0; meta->NextLevel = 0;
_spareMemory->Size = spareSize; _spareMemory->SizeForAllocator = spareSize;
_spareMemory->Position += size; _spareMemory->Position += size;
// insert the new chunk to position chain // insert the new chunk to position chain
int nearIdx = _spareMemory->PreviousNear; int nearIdx = _spareMemory->PreviousNear;
Metadata* near = &_spareMemory[nearIdx]; CacheLineMetadata* near = &_spareMemory[nearIdx];
near->NextNear = idx; near->NextNear = idx;
meta->PreviousNear = nearIdx; meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = idx; _spareMemory->PreviousNear = idx;
@@ -252,10 +252,10 @@ namespace StgSharp.HighPerformance.Memory
} }
private bool TryGetNewMetadata( private bool TryGetNewMetadata(
out Metadata* meta out CacheLineMetadata* meta
) )
{ {
ulong spareSize = _spareMemory->Size; ulong spareSize = _spareMemory->SizeForAllocator;
if (_metadataEnd >= MaxCacheLineCount) if (_metadataEnd >= MaxCacheLineCount)
{ {
meta = null; meta = null;
@@ -268,7 +268,7 @@ namespace StgSharp.HighPerformance.Memory
private bool TryTakeBucketEntry( private bool TryTakeBucketEntry(
ulong minimumSize, ulong minimumSize,
out Metadata* entry out CacheLineMetadata* entry
) )
{ {
int level = GetLevel(minimumSize); int level = GetLevel(minimumSize);
@@ -278,8 +278,8 @@ namespace StgSharp.HighPerformance.Memory
int idx = _bucket[level]; int idx = _bucket[level];
while (idx != 0) while (idx != 0)
{ {
Metadata* candidate = &_spareMemory[idx]; CacheLineMetadata* candidate = &_spareMemory[idx];
if (candidate->Size >= minimumSize) if (candidate->SizeForAllocator >= minimumSize)
{ {
RemoveFromBucket(level, candidate, idx); RemoveFromBucket(level, candidate, idx);
entry = candidate; entry = candidate;
@@ -293,7 +293,7 @@ namespace StgSharp.HighPerformance.Memory
int idx = _bucket[i]; int idx = _bucket[i];
if (idx != 0) if (idx != 0)
{ {
Metadata* candidate = &_spareMemory[idx]; CacheLineMetadata* candidate = &_spareMemory[idx];
RemoveFromBucketHead(i, candidate, idx); RemoveFromBucketHead(i, candidate, idx);
entry = candidate; entry = candidate;
return true; return true;
@@ -304,52 +304,6 @@ namespace StgSharp.HighPerformance.Memory
return false; return false;
} }
internal enum AllocationState : byte
{
SpareOrCollect = 0,
Free = 1,
Alloc = 2,
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(16)] private ulong PositionMask;
[FieldOffset(24)] private ulong SizeMask;
[FieldOffset(30)] internal AllocationState State;
[FieldOffset(31)] internal sbyte Level;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 Size
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
}
#region alloc and free #region alloc and free
public int Alloc( public int Alloc(
@@ -360,12 +314,12 @@ namespace StgSharp.HighPerformance.Memory
if (allocSize > _maxAllocSize) { if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size."); throw new OverflowException("Requested allocation size exceeds the maximum block size.");
} }
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) && if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
!TryAllocNewChunk(allocSize, out entry)) { !TryAllocNewChunk(allocSize, out entry)) {
throw new OverflowException("No available "); throw new OverflowException("No available ");
} }
int entryIdx = (int)(_spareMemory - entry); int entryIdx = (int)(_spareMemory - entry);
ulong entrySize = entry->Size; ulong entrySize = entry->SizeForAllocator;
if (entrySize < allocSize) { if (entrySize < allocSize) {
throw new OverflowException("Bucket returned an undersized block."); throw new OverflowException("Bucket returned an undersized block.");
} }
@@ -373,18 +327,18 @@ namespace StgSharp.HighPerformance.Memory
// slice tail // slice tail
if (remain > ulong.Min(4096UL, entrySize >> 7) && if (remain > ulong.Min(4096UL, entrySize >> 7) &&
TryGetNewMetadata(out Metadata* newEntry)) TryGetNewMetadata(out CacheLineMetadata* newEntry))
{ {
int newEntryIdx = (int)(_spareMemory - newEntry); int newEntryIdx = (int)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize; newEntry->Position = entry->Position + allocSize;
newEntry->Size = remain; newEntry->SizeForAllocator = remain;
newEntry->State = AllocationState.Free; newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain); int newLevel = GetLevel(remain);
int bucketHeadIdx = _bucket[newLevel]; int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
{ {
// if there is already an entry in the bucket, link it to the new entry // if there is already an entry in the bucket, link it to the new entry
Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = newEntryIdx; bucketHead->PreviousLevel = newEntryIdx;
} }
newEntry->NextLevel = bucketHeadIdx; newEntry->NextLevel = bucketHeadIdx;
@@ -394,7 +348,7 @@ namespace StgSharp.HighPerformance.Memory
// push to position chain // push to position chain
int afterIdx = entry->NextNear; int afterIdx = entry->NextNear;
Metadata* after = &_spareMemory[afterIdx]; CacheLineMetadata* after = &_spareMemory[afterIdx];
newEntry->NextNear = afterIdx; newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx; after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx; entry->NextNear = newEntryIdx;
@@ -416,7 +370,7 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
Metadata* current = &_spareMemory[currentIdx]; CacheLineMetadata* current = &_spareMemory[currentIdx];
if (current->State == AllocationState.Free) if (current->State == AllocationState.Free)
{ {
// scan next // scan next
@@ -424,24 +378,24 @@ namespace StgSharp.HighPerformance.Memory
int nextIdx = current->NextNear; int nextIdx = current->NextNear;
while (nextIdx != 0) while (nextIdx != 0)
{ {
Metadata* next = &_spareMemory[nextIdx]; CacheLineMetadata* next = &_spareMemory[nextIdx];
if (next->State != AllocationState.Free) if (next->State != AllocationState.Free)
{ {
break; break;
} }
size += next->Size; size += next->SizeForAllocator;
RecycleNode(next); RecycleNode(next);
nextIdx = current->NextNear; nextIdx = current->NextNear;
} }
if (size > 0) if (size > 0)
{ {
current->Size += size; current->SizeForAllocator += size;
RemoveFromBucket(current, currentIdx); RemoveFromBucket(current, currentIdx);
int newLevel = GetLevel(current->Size); int newLevel = GetLevel(current->SizeForAllocator);
int bucketHeadIdx = _bucket[newLevel]; int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
{ {
Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = currentIdx; bucketHead->PreviousLevel = currentIdx;
} }
current->NextLevel = bucketHeadIdx; current->NextLevel = bucketHeadIdx;
@@ -458,9 +412,9 @@ namespace StgSharp.HighPerformance.Memory
int collectScanWindow = 4 int collectScanWindow = 4
) )
{ {
Metadata* head = &_spareMemory[handle]; CacheLineMetadata* head = &_spareMemory[handle];
int headIdx = (int)(_spareMemory - head); int headIdx = (int)(_spareMemory - head);
Ptr64 size = head->Size; Ptr64 size = head->SizeForAllocator;
for (int i = 0; i < collectScanWindow; i++) for (int i = 0; i < collectScanWindow; i++)
{ {
// forward scan, previous direction // forward scan, previous direction
@@ -469,12 +423,12 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
Metadata* prev = &_spareMemory[forwardIdx]; CacheLineMetadata* prev = &_spareMemory[forwardIdx];
if (prev->State != AllocationState.Free) if (prev->State != AllocationState.Free)
{ {
break; break;
} }
size += prev->Size; size += prev->SizeForAllocator;
head->Position = prev->Position; head->Position = prev->Position;
RecycleNode(prev); RecycleNode(prev);
} }
@@ -486,25 +440,25 @@ namespace StgSharp.HighPerformance.Memory
{ {
break; break;
} }
Metadata* prev = &_spareMemory[nextIdx]; CacheLineMetadata* prev = &_spareMemory[nextIdx];
if (prev->State != AllocationState.Free) if (prev->State != AllocationState.Free)
{ {
break; break;
} }
size += prev->Size; size += prev->SizeForAllocator;
RecycleNode(prev); RecycleNode(prev);
} }
// recycle to level; // recycle to level;
head->Size = size; head->SizeForAllocator = size;
head->State = AllocationState.Free; head->State = AllocationState.Free;
head->PreviousLevel = 0; head->PreviousLevel = 0;
int level = GetLevel(head->Size); int level = GetLevel(head->SizeForAllocator);
int bucketHeadIdx = _bucket[level]; int bucketHeadIdx = _bucket[level];
head->NextLevel = 0; head->NextLevel = 0;
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
{ {
Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = headIdx; bucketHead->PreviousLevel = headIdx;
head->NextLevel = bucketHeadIdx; head->NextLevel = bucketHeadIdx;
} }
@@ -523,14 +477,14 @@ namespace StgSharp.HighPerformance.Memory
h = -1; h = -1;
return false; return false;
} }
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) && if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
!TryAllocNewChunk(allocSize, out entry)) !TryAllocNewChunk(allocSize, out entry))
{ {
h = -1; h = -1;
return false; return false;
} }
int entryIdx = (int)(_spareMemory - entry); int entryIdx = (int)(_spareMemory - entry);
ulong entrySize = entry->Size; ulong entrySize = entry->SizeForAllocator;
if (entrySize < allocSize) if (entrySize < allocSize)
{ {
h = -1; h = -1;
@@ -539,18 +493,18 @@ namespace StgSharp.HighPerformance.Memory
ulong remain = entrySize - allocSize; ulong remain = entrySize - allocSize;
// slice tail // slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry)) if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
{ {
int newEntryIdx = (int)(_spareMemory - newEntry); int newEntryIdx = (int)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize; newEntry->Position = entry->Position + allocSize;
newEntry->Size = remain; newEntry->SizeForAllocator = remain;
newEntry->State = AllocationState.Free; newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain); int newLevel = GetLevel(remain);
int bucketHeadIdx = _bucket[newLevel]; int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0) if (bucketHeadIdx != 0)
{ {
// if there is already an entry in the bucket, link it to the new entry // if there is already an entry in the bucket, link it to the new entry
Metadata* bucketHead = &_spareMemory[bucketHeadIdx]; CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
newEntry->NextLevel = bucketHeadIdx; newEntry->NextLevel = bucketHeadIdx;
bucketHead->PreviousLevel = newEntryIdx; bucketHead->PreviousLevel = newEntryIdx;
} else } else
@@ -562,7 +516,7 @@ namespace StgSharp.HighPerformance.Memory
// push to position chain // push to position chain
int afterIdx = entry->NextNear; int afterIdx = entry->NextNear;
Metadata* after = &_spareMemory[afterIdx]; CacheLineMetadata* after = &_spareMemory[afterIdx];
newEntry->NextNear = afterIdx; newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx; after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx; entry->NextNear = newEntryIdx;
StgSharp.Common/HighPerformance/Memory/L4.cs
+13 -14
View File
@@ -42,7 +42,7 @@ namespace StgSharp.HighPerformance.Memory
private const int PredictionLeastAhead = 1024; private const int PredictionLeastAhead = 1024;
// private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance; // private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
private readonly CacheLineHead* _head; private readonly CacheLineMetadata* _head;
private readonly MapArray* _mapCount; private readonly MapArray* _mapCount;
private readonly PredictionArray* _predictCount; private readonly PredictionArray* _predictCount;
private readonly CacheLineDescription* _prediction; private readonly CacheLineDescription* _prediction;
@@ -69,31 +69,30 @@ namespace StgSharp.HighPerformance.Memory
_bindedNuma = bindedNuma; _bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component. // init order is fallow by align requirement of each component.
nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize; ByteCapacity = normalizedBufferSize;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose. // malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance. // This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize); _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress; nuint ptr = _baseAddress;
AllocSize = entrySize;
// init all components with the buffer, and move the buffer pointer forward. // init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress + normalizedBufferSize; nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress); _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
ptr += entrySize; _buffer += normalizedBufferSize;
_head = (CacheLineHead*)ptr; _head = (CacheLineMetadata*)ptr;
ptr += headSize; ptr += headSize;
_prediction = (CacheLineDescription*)ptr; _prediction = (CacheLineDescription*)ptr;
ptr += predictionSize; ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { }); _map = SwissTable.Create(ptr, _ => { });
ptr += mapSize; ptr += mapCountSize;
_predictCount = (PredictionArray*)ptr; _predictCount = (PredictionArray*)ptr;
ptr += predictionCountSize; ptr += predictionCountSize;
_mapCount = (MapArray*)ptr; _mapCount = (MapArray*)ptr;
@@ -101,7 +100,7 @@ namespace StgSharp.HighPerformance.Memory
M512* mPtr = (M512*)_head; M512* mPtr = (M512*)_head;
for (uint i = 0; i < CacheLineCount; i++) { for (uint i = 0; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(i); mPtr->BroadCastFrom<ulong>(0);
} }
} }
@@ -141,7 +140,7 @@ namespace StgSharp.HighPerformance.Memory
// init order is fallow by align requirement of each component. // init order is fallow by align requirement of each component.
nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align
@@ -178,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle)) if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
{ {
// cache hit // cache hit
CacheLineHead* entry = (CacheLineHead*)entryHandle; CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0) if (Interlocked.Increment(ref entry->RefCount) > 0)
{ {
(*_predictCount)[entry->Id] = 0; (*_predictCount)[entry->Id] = 0;