working on L4's tlsf
This commit is contained in:
Youmu Liu
@@ -25,6 +25,7 @@
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
@@ -41,13 +42,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private unsafe class TLSF
|
||||
{
|
||||
|
||||
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
|
||||
public const int MaxCacheLineCount = 8192;
|
||||
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * Metadata.SizeOf;
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
|
||||
|
||||
private readonly int[] _bucket;
|
||||
private readonly Metadata* _spareMemory;
|
||||
private readonly CacheLineMetadata* _spareMemory;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
@@ -65,17 +65,17 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_totalSize = byteSize;
|
||||
_basePtr = (Ptr64)(ulong)externalPointer;
|
||||
_align = 4096; // align of a memory page
|
||||
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
|
||||
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, CacheLineMetadata.Mask48Bits)));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new int[GetLevel(_maxAllocSize) + 2];
|
||||
_spareMemory = (Metadata*)(ulong)(_basePtr + _totalSize);
|
||||
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
|
||||
_metadataEnd = 0;
|
||||
|
||||
// init spare memory node
|
||||
|
||||
_spareMemory->Position = _basePtr;
|
||||
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
|
||||
_spareMemory->SizeForSpare = _totalSize - CacheLineMetadata.SizeOf;
|
||||
_spareMemory->NextLevel = 0;
|
||||
_spareMemory->NextNear = 0;
|
||||
_spareMemory->PreviousLevel = 0;
|
||||
@@ -98,11 +98,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
///
|
||||
/// </param>
|
||||
private void RecycleNode(
|
||||
Metadata* node
|
||||
CacheLineMetadata* node
|
||||
)
|
||||
{
|
||||
int nodeIdx = (int)(_spareMemory - node);
|
||||
int level = GetLevel(node->Size);
|
||||
int level = GetLevel(node->SizeForAllocator);
|
||||
int nextLevelIdx = node->NextLevel;
|
||||
int prevLevelIdx = node->PreviousLevel;
|
||||
RemoveFromBucket(level, node, nodeIdx);
|
||||
@@ -110,11 +110,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
// remove from position chain
|
||||
int prevNearIdx = node->PreviousNear;
|
||||
int nextNearIdx = node->NextNear;
|
||||
Metadata* prevNear = &_spareMemory[prevNearIdx];
|
||||
CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
|
||||
prevNear->NextNear = nextNearIdx;
|
||||
if (nextNearIdx != 0)
|
||||
{
|
||||
Metadata* nextNear = &_spareMemory[nextNearIdx];
|
||||
CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
|
||||
nextNear->PreviousNear = prevNearIdx;
|
||||
} else
|
||||
{
|
||||
@@ -131,7 +131,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int nextIdx = _spareMemory->NextLevel;
|
||||
if (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[nextIdx];
|
||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
||||
next->PreviousLevel = nodeIdx;
|
||||
}
|
||||
node->NextLevel = nextIdx;
|
||||
@@ -140,16 +140,16 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RemoveFromBucket(
|
||||
Metadata* node,
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
RemoveFromBucket(GetLevel(node->Size), node, nodeIdx);
|
||||
RemoveFromBucket(GetLevel(node->SizeForAllocator), node, nodeIdx);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(
|
||||
int level,
|
||||
Metadata* node,
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
@@ -162,12 +162,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
} else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
@@ -177,7 +177,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
// Assuming node to remove is head f bucket
|
||||
private void RemoveFromBucketHead(
|
||||
int level,
|
||||
Metadata* node,
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
@@ -199,7 +199,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
/**/
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = 0;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
@@ -208,11 +208,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
private bool TryAllocNewChunk(
|
||||
ulong size,
|
||||
out Metadata* meta
|
||||
out CacheLineMetadata* meta
|
||||
)
|
||||
{
|
||||
int idx = _spareMemory->NextLevel;
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
ulong spareSize = _spareMemory->SizeForAllocator;
|
||||
if (idx != 0)
|
||||
{
|
||||
if (spareSize < size)
|
||||
@@ -224,7 +224,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int nextSpareIdx = meta->NextLevel;
|
||||
if (nextSpareIdx != 0)
|
||||
{
|
||||
Metadata* nextSpare = &_spareMemory[nextSpareIdx];
|
||||
CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
|
||||
nextSpare->PreviousLevel = 0;
|
||||
}
|
||||
_spareMemory->NextLevel = nextSpareIdx;
|
||||
@@ -233,17 +233,17 @@ namespace StgSharp.HighPerformance.Memory
|
||||
return false;
|
||||
}
|
||||
spareSize -= size;
|
||||
meta->Size = size;
|
||||
meta->SizeForAllocator = size;
|
||||
meta->Position = _spareMemory->Position;
|
||||
meta->State = AllocationState.Free;
|
||||
meta->PreviousLevel = 0;
|
||||
meta->NextLevel = 0;
|
||||
_spareMemory->Size = spareSize;
|
||||
_spareMemory->SizeForAllocator = spareSize;
|
||||
_spareMemory->Position += size;
|
||||
|
||||
// insert the new chunk to position chain
|
||||
int nearIdx = _spareMemory->PreviousNear;
|
||||
Metadata* near = &_spareMemory[nearIdx];
|
||||
CacheLineMetadata* near = &_spareMemory[nearIdx];
|
||||
near->NextNear = idx;
|
||||
meta->PreviousNear = nearIdx;
|
||||
_spareMemory->PreviousNear = idx;
|
||||
@@ -252,10 +252,10 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
|
||||
private bool TryGetNewMetadata(
|
||||
out Metadata* meta
|
||||
out CacheLineMetadata* meta
|
||||
)
|
||||
{
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
ulong spareSize = _spareMemory->SizeForAllocator;
|
||||
if (_metadataEnd >= MaxCacheLineCount)
|
||||
{
|
||||
meta = null;
|
||||
@@ -268,7 +268,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
private bool TryTakeBucketEntry(
|
||||
ulong minimumSize,
|
||||
out Metadata* entry
|
||||
out CacheLineMetadata* entry
|
||||
)
|
||||
{
|
||||
int level = GetLevel(minimumSize);
|
||||
@@ -278,8 +278,8 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int idx = _bucket[level];
|
||||
while (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[idx];
|
||||
if (candidate->Size >= minimumSize)
|
||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
||||
if (candidate->SizeForAllocator >= minimumSize)
|
||||
{
|
||||
RemoveFromBucket(level, candidate, idx);
|
||||
entry = candidate;
|
||||
@@ -293,7 +293,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[idx];
|
||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
||||
RemoveFromBucketHead(i, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
@@ -304,52 +304,6 @@ namespace StgSharp.HighPerformance.Memory
|
||||
return false;
|
||||
}
|
||||
|
||||
internal enum AllocationState : byte
|
||||
{
|
||||
|
||||
SpareOrCollect = 0,
|
||||
Free = 1,
|
||||
Alloc = 2,
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||
internal struct Metadata
|
||||
{
|
||||
|
||||
[FieldOffset(16)] private ulong PositionMask;
|
||||
[FieldOffset(24)] private ulong SizeMask;
|
||||
|
||||
[FieldOffset(30)] internal AllocationState State;
|
||||
[FieldOffset(31)] internal sbyte Level;
|
||||
|
||||
[FieldOffset(0)] public int NextLevel;
|
||||
[FieldOffset(4)] public int NextNear;
|
||||
[FieldOffset(8)] public int PreviousLevel;
|
||||
[FieldOffset(12)] public int PreviousNear;
|
||||
|
||||
public Ptr64 Position
|
||||
{
|
||||
readonly get => PositionMask;
|
||||
set => PositionMask = value;
|
||||
}
|
||||
|
||||
public Ptr64 Size
|
||||
{
|
||||
readonly get => (Ptr64)(SizeMask & Mask48Bits);
|
||||
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
|
||||
}
|
||||
|
||||
public Ptr64 SizeForSpare
|
||||
{
|
||||
readonly get => SizeMask;
|
||||
set => SizeMask = value;
|
||||
}
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
|
||||
|
||||
}
|
||||
|
||||
#region alloc and free
|
||||
|
||||
public int Alloc(
|
||||
@@ -360,12 +314,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
if (allocSize > _maxAllocSize) {
|
||||
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry)) {
|
||||
throw new OverflowException("No available ");
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
ulong entrySize = entry->SizeForAllocator;
|
||||
if (entrySize < allocSize) {
|
||||
throw new OverflowException("Bucket returned an undersized block.");
|
||||
}
|
||||
@@ -373,18 +327,18 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
// slice tail
|
||||
if (remain > ulong.Min(4096UL, entrySize >> 7) &&
|
||||
TryGetNewMetadata(out Metadata* newEntry))
|
||||
TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->SizeForAllocator = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
}
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
@@ -394,7 +348,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[afterIdx];
|
||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
@@ -416,7 +370,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* current = &_spareMemory[currentIdx];
|
||||
CacheLineMetadata* current = &_spareMemory[currentIdx];
|
||||
if (current->State == AllocationState.Free)
|
||||
{
|
||||
// scan next
|
||||
@@ -424,24 +378,24 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int nextIdx = current->NextNear;
|
||||
while (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[nextIdx];
|
||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
||||
if (next->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += next->Size;
|
||||
size += next->SizeForAllocator;
|
||||
RecycleNode(next);
|
||||
nextIdx = current->NextNear;
|
||||
}
|
||||
if (size > 0)
|
||||
{
|
||||
current->Size += size;
|
||||
current->SizeForAllocator += size;
|
||||
RemoveFromBucket(current, currentIdx);
|
||||
int newLevel = GetLevel(current->Size);
|
||||
int newLevel = GetLevel(current->SizeForAllocator);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = currentIdx;
|
||||
}
|
||||
current->NextLevel = bucketHeadIdx;
|
||||
@@ -458,9 +412,9 @@ namespace StgSharp.HighPerformance.Memory
|
||||
int collectScanWindow = 4
|
||||
)
|
||||
{
|
||||
Metadata* head = &_spareMemory[handle];
|
||||
CacheLineMetadata* head = &_spareMemory[handle];
|
||||
int headIdx = (int)(_spareMemory - head);
|
||||
Ptr64 size = head->Size;
|
||||
Ptr64 size = head->SizeForAllocator;
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// forward scan, previous direction
|
||||
@@ -469,12 +423,12 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[forwardIdx];
|
||||
CacheLineMetadata* prev = &_spareMemory[forwardIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
size += prev->SizeForAllocator;
|
||||
head->Position = prev->Position;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
@@ -486,25 +440,25 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[nextIdx];
|
||||
CacheLineMetadata* prev = &_spareMemory[nextIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
size += prev->SizeForAllocator;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
|
||||
// recycle to level;
|
||||
head->Size = size;
|
||||
head->SizeForAllocator = size;
|
||||
head->State = AllocationState.Free;
|
||||
head->PreviousLevel = 0;
|
||||
int level = GetLevel(head->Size);
|
||||
int level = GetLevel(head->SizeForAllocator);
|
||||
int bucketHeadIdx = _bucket[level];
|
||||
head->NextLevel = 0;
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = headIdx;
|
||||
head->NextLevel = bucketHeadIdx;
|
||||
}
|
||||
@@ -523,14 +477,14 @@ namespace StgSharp.HighPerformance.Memory
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry))
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
ulong entrySize = entry->SizeForAllocator;
|
||||
if (entrySize < allocSize)
|
||||
{
|
||||
h = -1;
|
||||
@@ -539,18 +493,18 @@ namespace StgSharp.HighPerformance.Memory
|
||||
ulong remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->SizeForAllocator = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
} else
|
||||
@@ -562,7 +516,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[afterIdx];
|
||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
|
||||
Reference in New Issue
Block a user