working on L4's tlsf
This commit is contained in:
@@ -65,11 +65,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
#region hlsf entry alias
|
||||
|
||||
[FieldOffset(8)] public readonly Ptr64 Address;
|
||||
[FieldOffset(16)] public readonly Ptr64 Size;
|
||||
public readonly Ptr64 Address => AllocatorEntry.Address;
|
||||
|
||||
public readonly Ptr64 Size => AllocatorEntry.Size;
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -117,24 +117,5 @@ namespace StgSharp.HighPerformance.Memory
|
||||
return false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe bool TryGetEmptyLine(
|
||||
nuint size,
|
||||
out int index
|
||||
|
||||
)
|
||||
{
|
||||
if (!_bufferAllocator.TryAlloc(size, out Tlsf.Handle handle))
|
||||
{
|
||||
index = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
// origin :CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
|
||||
CacheLineHead* entry = (CacheLineHead*)handle.Address;
|
||||
index = entry->Id;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,7 +125,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
int retryCount = 0, remain = missIndex - i;
|
||||
const int maxRetry = 1024;
|
||||
while (!TryGetEmptyLine(pHandle.Size, out index))
|
||||
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
|
||||
{
|
||||
remain = Evict(remain);
|
||||
retryCount++;
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="L4.Slab"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Tlsf = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private sealed unsafe class Slab : SlabAllocator<Tlsf.Metadata>
|
||||
{
|
||||
|
||||
private const int Capacity = L4.CacheLineCount;
|
||||
|
||||
private int _maxUnused;
|
||||
private int _recycleCount;
|
||||
|
||||
internal readonly CacheLineHead* _basePtr;
|
||||
internal readonly int* _indexReuse;
|
||||
|
||||
public Slab(
|
||||
nuint ptr
|
||||
)
|
||||
{
|
||||
_basePtr = (CacheLineHead*)ptr;
|
||||
_indexReuse = (int*)(_basePtr + Capacity);
|
||||
_maxUnused = 0;
|
||||
_recycleCount = 0;
|
||||
}
|
||||
|
||||
public static nuint RequestedAllocation { get; } = ((nuint)(Unsafe.SizeOf<CacheLineHead>() + sizeof(int))) * Capacity;
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycleCount > 0) {
|
||||
return (nuint)(_basePtr + _indexReuse[_recycleCount - 1]);
|
||||
}
|
||||
if (_maxUnused >= Capacity) {
|
||||
throw new OverflowException("Slab is full");
|
||||
}
|
||||
int index = _maxUnused;
|
||||
_maxUnused++;
|
||||
return (nuint)(_basePtr + index);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_indexReuse[_recycleCount] = (int)index;
|
||||
_recycleCount++;
|
||||
}
|
||||
|
||||
public override SlabAllocationHandle<Tlsf.Metadata> ReadAllocation()
|
||||
{
|
||||
return new(this, false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,585 @@
|
||||
//-----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
// file="L4.Tlsf"
|
||||
// Project: StgSharp
|
||||
// AuthorGroup: Nitload
|
||||
// Copyright (c) Nitload. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
//
|
||||
// -----------------------------------------------------------------------
|
||||
// -----------------------------------------------------------------------
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private unsafe class TLSF
|
||||
{
|
||||
|
||||
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
|
||||
public const int MaxCacheLineCount = 8192;
|
||||
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * Metadata.SizeOf;
|
||||
|
||||
private readonly int[] _bucket;
|
||||
private readonly Metadata* _spareMemory;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
private int _metadataEnd;
|
||||
private readonly Ptr64 _basePtr;
|
||||
private readonly ulong _maxAllocSize;
|
||||
private readonly ulong _totalSize;
|
||||
|
||||
internal TLSF(
|
||||
nuint byteSize,
|
||||
ulong maxBlockSize,
|
||||
void* externalPointer
|
||||
)
|
||||
{
|
||||
_totalSize = byteSize;
|
||||
_basePtr = (Ptr64)(ulong)externalPointer;
|
||||
_align = 4096; // align of a memory page
|
||||
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new int[GetLevel(_maxAllocSize) + 2];
|
||||
_spareMemory = (Metadata*)(ulong)(_basePtr + _totalSize);
|
||||
_metadataEnd = 0;
|
||||
|
||||
// init spare memory node
|
||||
|
||||
_spareMemory->Position = _basePtr;
|
||||
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
|
||||
_spareMemory->NextLevel = 0;
|
||||
_spareMemory->NextNear = 0;
|
||||
_spareMemory->PreviousLevel = 0;
|
||||
_spareMemory->PreviousNear = 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetLevel(
|
||||
ulong size
|
||||
)
|
||||
{
|
||||
int origin = 57 - BitOperations.LeadingZeroCount(size);
|
||||
return int.Min(origin, _maxLevel);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Put a metadata node back to free list and waiting for next allocation.
|
||||
/// </summary>
|
||||
/// <param name="node">
|
||||
///
|
||||
/// </param>
|
||||
private void RecycleNode(
|
||||
Metadata* node
|
||||
)
|
||||
{
|
||||
int nodeIdx = (int)(_spareMemory - node);
|
||||
int level = GetLevel(node->Size);
|
||||
int nextLevelIdx = node->NextLevel;
|
||||
int prevLevelIdx = node->PreviousLevel;
|
||||
RemoveFromBucket(level, node, nodeIdx);
|
||||
|
||||
// remove from position chain
|
||||
int prevNearIdx = node->PreviousNear;
|
||||
int nextNearIdx = node->NextNear;
|
||||
Metadata* prevNear = &_spareMemory[prevNearIdx];
|
||||
prevNear->NextNear = nextNearIdx;
|
||||
if (nextNearIdx != 0)
|
||||
{
|
||||
Metadata* nextNear = &_spareMemory[nextNearIdx];
|
||||
nextNear->PreviousNear = prevNearIdx;
|
||||
} else
|
||||
{
|
||||
_spareMemory->PreviousNear = prevNearIdx;
|
||||
}
|
||||
node->PreviousNear = 0;
|
||||
node->NextNear = 0;
|
||||
|
||||
node->Position = 0;
|
||||
node->SizeForSpare = 0;
|
||||
node->State = AllocationState.SpareOrCollect;
|
||||
node->PreviousLevel = 0;
|
||||
|
||||
int nextIdx = _spareMemory->NextLevel;
|
||||
if (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[nextIdx];
|
||||
next->PreviousLevel = nodeIdx;
|
||||
}
|
||||
node->NextLevel = nextIdx;
|
||||
_spareMemory->NextLevel = nodeIdx;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RemoveFromBucket(
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
RemoveFromBucket(GetLevel(node->Size), node, nodeIdx);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(
|
||||
int level,
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
int currentNextLevelIdx = node->NextLevel;
|
||||
int currentPrevLevelIdx = node->PreviousLevel;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
} else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
// Assuming node to remove is head f bucket
|
||||
private void RemoveFromBucketHead(
|
||||
int level,
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
int currentNextLevelIdx = node->NextLevel;
|
||||
|
||||
// int currentPrevLevelIdx = 0;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
}
|
||||
/*
|
||||
else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
/**/
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = 0;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
private bool TryAllocNewChunk(
|
||||
ulong size,
|
||||
out Metadata* meta
|
||||
)
|
||||
{
|
||||
int idx = _spareMemory->NextLevel;
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
if (idx != 0)
|
||||
{
|
||||
if (spareSize < size)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
meta = &_spareMemory[idx];
|
||||
int nextSpareIdx = meta->NextLevel;
|
||||
if (nextSpareIdx != 0)
|
||||
{
|
||||
Metadata* nextSpare = &_spareMemory[nextSpareIdx];
|
||||
nextSpare->PreviousLevel = 0;
|
||||
}
|
||||
_spareMemory->NextLevel = nextSpareIdx;
|
||||
} else
|
||||
if (!TryGetNewMetadata(out meta)) {
|
||||
return false;
|
||||
}
|
||||
spareSize -= size;
|
||||
meta->Size = size;
|
||||
meta->Position = _spareMemory->Position;
|
||||
meta->State = AllocationState.Free;
|
||||
meta->PreviousLevel = 0;
|
||||
meta->NextLevel = 0;
|
||||
_spareMemory->Size = spareSize;
|
||||
_spareMemory->Position += size;
|
||||
|
||||
// insert the new chunk to position chain
|
||||
int nearIdx = _spareMemory->PreviousNear;
|
||||
Metadata* near = &_spareMemory[nearIdx];
|
||||
near->NextNear = idx;
|
||||
meta->PreviousNear = nearIdx;
|
||||
_spareMemory->PreviousNear = idx;
|
||||
meta->NextNear = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryGetNewMetadata(
|
||||
out Metadata* meta
|
||||
)
|
||||
{
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
if (_metadataEnd >= MaxCacheLineCount)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
_metadataEnd++;
|
||||
meta = &_spareMemory[_metadataEnd];
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryTakeBucketEntry(
|
||||
ulong minimumSize,
|
||||
out Metadata* entry
|
||||
)
|
||||
{
|
||||
int level = GetLevel(minimumSize);
|
||||
|
||||
// first try to find in the target level
|
||||
{
|
||||
int idx = _bucket[level];
|
||||
while (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[idx];
|
||||
if (candidate->Size >= minimumSize)
|
||||
{
|
||||
RemoveFromBucket(level, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
idx = candidate->NextLevel;
|
||||
}
|
||||
}
|
||||
for (int i = level + 1; i < _bucket.Length; i++)
|
||||
{
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[idx];
|
||||
RemoveFromBucketHead(i, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
internal enum AllocationState : byte
|
||||
{
|
||||
|
||||
SpareOrCollect = 0,
|
||||
Free = 1,
|
||||
Alloc = 2,
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||
internal struct Metadata
|
||||
{
|
||||
|
||||
[FieldOffset(16)] private ulong PositionMask;
|
||||
[FieldOffset(24)] private ulong SizeMask;
|
||||
|
||||
[FieldOffset(30)] internal AllocationState State;
|
||||
[FieldOffset(31)] internal sbyte Level;
|
||||
|
||||
[FieldOffset(0)] public int NextLevel;
|
||||
[FieldOffset(4)] public int NextNear;
|
||||
[FieldOffset(8)] public int PreviousLevel;
|
||||
[FieldOffset(12)] public int PreviousNear;
|
||||
|
||||
public Ptr64 Position
|
||||
{
|
||||
readonly get => PositionMask;
|
||||
set => PositionMask = value;
|
||||
}
|
||||
|
||||
public Ptr64 Size
|
||||
{
|
||||
readonly get => (Ptr64)(SizeMask & Mask48Bits);
|
||||
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
|
||||
}
|
||||
|
||||
public Ptr64 SizeForSpare
|
||||
{
|
||||
readonly get => SizeMask;
|
||||
set => SizeMask = value;
|
||||
}
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
|
||||
|
||||
}
|
||||
|
||||
#region alloc and free
|
||||
|
||||
public int Alloc(
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
|
||||
if (allocSize > _maxAllocSize) {
|
||||
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry)) {
|
||||
throw new OverflowException("No available ");
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
if (entrySize < allocSize) {
|
||||
throw new OverflowException("Bucket returned an undersized block.");
|
||||
}
|
||||
ulong remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > ulong.Min(4096UL, entrySize >> 7) &&
|
||||
TryGetNewMetadata(out Metadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
}
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->Position - (ulong)_basePtr > _totalSize) {
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
return entryIdx;
|
||||
}
|
||||
|
||||
public void Collect()
|
||||
{
|
||||
int currentIdx = _spareMemory->NextNear;
|
||||
do
|
||||
{
|
||||
if (currentIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* current = &_spareMemory[currentIdx];
|
||||
if (current->State == AllocationState.Free)
|
||||
{
|
||||
// scan next
|
||||
Ptr64 size = 0;
|
||||
int nextIdx = current->NextNear;
|
||||
while (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[nextIdx];
|
||||
if (next->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += next->Size;
|
||||
RecycleNode(next);
|
||||
nextIdx = current->NextNear;
|
||||
}
|
||||
if (size > 0)
|
||||
{
|
||||
current->Size += size;
|
||||
RemoveFromBucket(current, currentIdx);
|
||||
int newLevel = GetLevel(current->Size);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = currentIdx;
|
||||
}
|
||||
current->NextLevel = bucketHeadIdx;
|
||||
current->PreviousLevel = 0;
|
||||
_bucket[newLevel] = currentIdx;
|
||||
}
|
||||
}
|
||||
currentIdx = current->NextNear;
|
||||
} while (true);
|
||||
}
|
||||
|
||||
public void Free(
|
||||
int handle,
|
||||
int collectScanWindow = 4
|
||||
)
|
||||
{
|
||||
Metadata* head = &_spareMemory[handle];
|
||||
int headIdx = (int)(_spareMemory - head);
|
||||
Ptr64 size = head->Size;
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// forward scan, previous direction
|
||||
int forwardIdx = head->PreviousNear;
|
||||
if (forwardIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[forwardIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
head->Position = prev->Position;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// backward scan, next direction
|
||||
int nextIdx = head->NextNear;
|
||||
if (nextIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[nextIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
|
||||
// recycle to level;
|
||||
head->Size = size;
|
||||
head->State = AllocationState.Free;
|
||||
head->PreviousLevel = 0;
|
||||
int level = GetLevel(head->Size);
|
||||
int bucketHeadIdx = _bucket[level];
|
||||
head->NextLevel = 0;
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = headIdx;
|
||||
head->NextLevel = bucketHeadIdx;
|
||||
}
|
||||
_bucket[level] = headIdx;
|
||||
return;
|
||||
}
|
||||
|
||||
public bool TryAlloc(
|
||||
nuint size,
|
||||
out int h
|
||||
)
|
||||
{
|
||||
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
|
||||
if (allocSize > _maxAllocSize)
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry))
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
if (entrySize < allocSize)
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
ulong remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
} else
|
||||
{
|
||||
newEntry->NextLevel = 0;
|
||||
}
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->Position - (ulong)_basePtr > _totalSize)
|
||||
{
|
||||
h = default;
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
h = entryIdx;
|
||||
return true;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -31,8 +31,6 @@ using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System.Data;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
using HLSF = StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4 : IDisposable
|
||||
@@ -50,8 +48,6 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private readonly CacheLineDescription* _prediction;
|
||||
private bool _disposed;
|
||||
private bool _exhausted;
|
||||
|
||||
private readonly HLSF _bufferAllocator;
|
||||
private int _activeCount;
|
||||
|
||||
private int _aheadCount;
|
||||
@@ -59,9 +55,10 @@ namespace StgSharp.HighPerformance.Memory
|
||||
private readonly int _bindedNuma;
|
||||
private volatile int _prefetching;
|
||||
private readonly nuint _baseAddress;
|
||||
private readonly Slab _entryManager;
|
||||
private readonly SwissTable _map;
|
||||
|
||||
private readonly TLSF _bufferAllocator;
|
||||
|
||||
public L4(
|
||||
int bindedNuma,
|
||||
nuint bufferSize = 8192 * 8192,
|
||||
@@ -72,7 +69,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
_bindedNuma = bindedNuma;
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
|
||||
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
@@ -89,9 +86,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
nuint _buffer = _baseAddress + normalizedBufferSize;
|
||||
_entryManager = new Slab(_buffer);
|
||||
_bufferAllocator = new HLSF(entrySize, align, maxCacheLineSize, (void*)_baseAddress,
|
||||
static _ => { }/*, _entryManager/**/);
|
||||
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
|
||||
ptr += entrySize;
|
||||
_head = (CacheLineHead*)ptr;
|
||||
ptr += headSize;
|
||||
@@ -141,12 +136,11 @@ namespace StgSharp.HighPerformance.Memory
|
||||
}
|
||||
_disposed = true;
|
||||
_map.Dispose();
|
||||
_entryManager.Dispose();
|
||||
_bufferAllocator.Dispose();
|
||||
GC.SuppressFinalize(_bufferAllocator);
|
||||
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint entrySize = Slab.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint entrySize = TLSF.RequestedAllocation; // SIMDID, no less than 16 byte bit
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
|
||||
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
|
||||
@@ -382,7 +382,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
|
||||
[FieldOffset(0)] internal readonly Metadata* _meta;
|
||||
[FieldOffset(8)] public readonly void* Address;
|
||||
[FieldOffset(8)] public readonly Ptr64 Address;
|
||||
[FieldOffset(16)] public readonly Ptr64 Size;
|
||||
|
||||
internal Handle(
|
||||
@@ -391,7 +391,7 @@ namespace StgSharp.HighPerformance.Memory
|
||||
)
|
||||
{
|
||||
_meta = meta;
|
||||
Address = (void*)(ulong)meta->Position;
|
||||
Address = (ulong)meta->Position;
|
||||
Size = meta->Size;
|
||||
}
|
||||
|
||||
@@ -400,13 +400,13 @@ namespace StgSharp.HighPerformance.Memory
|
||||
|
||||
public ref byte this[
|
||||
int offset
|
||||
] => ref ((byte*)Address)[offset];
|
||||
] => ref ((byte*)(ulong)Address)[offset];
|
||||
|
||||
public Span<byte> AsSpan(
|
||||
int range = -1
|
||||
)
|
||||
{
|
||||
return new Span<byte>(Address, range < 0 ? (int)(ulong)Size : range);
|
||||
return new Span<byte>((byte*)(ulong)Address, range < 0 ? (int)(ulong)Size : range);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user