predict method of l4
This commit is contained in:
@@ -6,6 +6,7 @@
|
|||||||
// -----------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------
|
||||||
|
|
||||||
using System.Runtime.InteropServices;
|
using System.Runtime.InteropServices;
|
||||||
|
using System.Xml.Serialization;
|
||||||
|
|
||||||
namespace Nitload.Common.HighPerformance.Memory
|
namespace Nitload.Common.HighPerformance.Memory
|
||||||
{
|
{
|
||||||
@@ -27,10 +28,38 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
|
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
|
||||||
|
|
||||||
[FieldOffset(16)] internal AllocationState State;
|
[FieldOffset(31)] internal AllocationState State;
|
||||||
|
|
||||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
|
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// only available in L4.Predict()
|
||||||
|
/// </summary>
|
||||||
|
public void CheckCount()
|
||||||
|
{
|
||||||
|
uint count = Volatile.Read(ref ConsumeCount);
|
||||||
|
if (ushort.MaxValue < count)
|
||||||
|
{
|
||||||
|
uint delta = unchecked(0u - count);
|
||||||
|
_ = Interlocked.Add(ref ConsumeCount, delta);
|
||||||
|
_ = Interlocked.Add(ref PredictCount, delta);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void IncreaseConsumeCount()
|
||||||
|
{
|
||||||
|
unchecked {
|
||||||
|
_ = Interlocked.Increment(ref ConsumeCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void IncreasePredictCount()
|
||||||
|
{
|
||||||
|
unchecked {
|
||||||
|
_ = Interlocked.Increment(ref PredictCount);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#region tlsf linked list
|
#region tlsf linked list
|
||||||
|
|
||||||
[FieldOffset(0)] public short NextLevel;
|
[FieldOffset(0)] public short NextLevel;
|
||||||
@@ -42,8 +71,15 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
#region buffer
|
#region buffer
|
||||||
|
|
||||||
[FieldOffset(8)] public uint PositionMask;
|
[FieldOffset(8)] public ulong Position;
|
||||||
[FieldOffset(12)] public uint SizeMask;
|
[FieldOffset(16)] public uint SizeMask;
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region count
|
||||||
|
|
||||||
|
[FieldOffset(20)] public uint PredictCount;
|
||||||
|
[FieldOffset(24)] public uint ConsumeCount;
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,8 +72,8 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
|
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
|
||||||
|
|
||||||
// process metadata, unfinished
|
// process metadata, unfinished
|
||||||
uint offset = metadata->PositionMask;
|
ulong offset = metadata->Position;
|
||||||
nuint address = /*base ptr + */ offset;
|
nuint address = /*base ptr + */ (nuint)offset;
|
||||||
_ = Interlocked.Decrement(ref _predict_count);
|
_ = Interlocked.Decrement(ref _predict_count);
|
||||||
|
|
||||||
return new Handle(address, (nuint)handle);
|
return new Handle(address, (nuint)handle);
|
||||||
|
|||||||
@@ -18,8 +18,57 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
internal partial int Predict()
|
internal partial int Predict()
|
||||||
{
|
{
|
||||||
throw new NotImplementedException();
|
if (!TryGetCurrentPredictor(out IL4Predictor? p)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
int total_count = 0, count;
|
||||||
|
Span<L4.AddressSource> as_sp = stackalloc L4.AddressSource[4];
|
||||||
|
Predict:
|
||||||
|
while ((count = p.PredictMission(in as_sp)) > 0)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
{
|
||||||
|
if (_addressMap.TryGet(as_sp[i], out ulong map_handle_adr))
|
||||||
|
{
|
||||||
|
L4.CacheLineMetadata* map_handle = (L4.CacheLineMetadata*)map_handle_adr;
|
||||||
|
map_handle->IncreasePredictCount();
|
||||||
|
map_handle->CheckCount();
|
||||||
|
} else
|
||||||
|
{
|
||||||
|
uint size = as_sp[i].Size;
|
||||||
|
int handle_idx = _bufferAllocator.Alloc(size);
|
||||||
|
L4.CacheLineMetadata* handle = _bufferAllocator.MetadataAt(handle_idx);
|
||||||
|
if (!_addressMap.TryAddOrSet(as_sp[i], (ulong)handle)) {
|
||||||
|
throw new InvalidOperationException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total_count += count;
|
||||||
|
if (total_count >= 2048) {
|
||||||
|
return total_count;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (MoveNextPredictor())
|
||||||
|
{
|
||||||
|
_ = TryGetCurrentPredictor(out p);
|
||||||
|
goto Predict;
|
||||||
|
}
|
||||||
|
return total_count;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
goto 本身没问题,但当前控制流有几个需要先确认的点:
|
||||||
|
|
||||||
|
高:换 predictor 的分支看起来反了。 MoveNextPredictor() 在 _predict 到达 _enter 时返回 true。因此 L4.Predict.cs:51 现在会在队列耗尽时再次取 predictor;TryGetCurrentPredictor 失败后仍 goto Predict,接着会用空的 p 调 PredictMission。反过来,有下一个 predictor 时它会提前返回,不再处理后续 predictor。
|
||||||
|
|
||||||
|
高:新分配的 metadata 没有增加预测计数。 现有映射分支会调用 IncreasePredictCount(),但新分配分支只分配并写入 Swiss。若该计数表示未消费预测数,新条目也需要相同的计数处理;还要确认 TLSF 重用 metadata 时会重置旧计数。
|
||||||
|
|
||||||
|
高:同一轮内重复的 AddressSource 可能重复分配。 SwissTable.TryGet 只查 published buffer,而新条目先写入 inactive buffer。后续再次预测相同 key 时会查不到,重新分配 metadata,再由 TryAddOrSet 覆盖先前 value,旧分配就失去映射。
|
||||||
|
|
||||||
|
中:需要校验 PredictMission 的返回值范围。 Span 容量是 4;count > 4 会让 as_sp[i] 越界,负值则被当作本轮结束。建议契约明确要求 0..4,并在 L4 边界验证。
|
||||||
|
|
||||||
|
本轮未改代码。L4.Predict.cs 的文件诊断通过,但尚未运行完整构建或运行时测试。
|
||||||
|
*/
|
||||||
@@ -16,7 +16,6 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
public partial class L4
|
public partial class L4
|
||||||
{
|
{
|
||||||
|
|
||||||
private readonly IL4Predictor[] _ring = new IL4Predictor[256];
|
|
||||||
private volatile int _consume;
|
private volatile int _consume;
|
||||||
private volatile int _enter;
|
private volatile int _enter;
|
||||||
private volatile int _predict;
|
private volatile int _predict;
|
||||||
@@ -45,7 +44,8 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
if (ReferenceEquals(predictor, _ring[idx]))
|
if (ReferenceEquals(predictor, _ring[idx]))
|
||||||
{
|
{
|
||||||
int end = _predict;
|
int end = _predict;
|
||||||
for (int move = i; move < end - 1; move++) {
|
for (int move = i; move < end - 1; move++)
|
||||||
|
{
|
||||||
_ring[(byte)move] = _ring[(byte)(move + 1)];
|
_ring[(byte)move] = _ring[(byte)(move + 1)];
|
||||||
}
|
}
|
||||||
_ring[(byte)(end - 1)] = default!;
|
_ring[(byte)(end - 1)] = default!;
|
||||||
@@ -61,6 +61,12 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal bool MoveNextPredictor()
|
||||||
|
{
|
||||||
|
int cur = Interlocked.Increment(ref _predict);
|
||||||
|
return cur == _enter;
|
||||||
|
}
|
||||||
|
|
||||||
internal void RegistPredictor(
|
internal void RegistPredictor(
|
||||||
IL4Predictor predictor
|
IL4Predictor predictor
|
||||||
)
|
)
|
||||||
@@ -69,18 +75,25 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
SpinWait sp = new();
|
SpinWait sp = new();
|
||||||
|
|
||||||
|
wait_for_spare:
|
||||||
|
WaitForSpare();
|
||||||
|
|
||||||
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
|
||||||
sp.SpinOnce();
|
sp.SpinOnce();
|
||||||
}
|
}
|
||||||
|
if (_enter - _consume >= PredictorRingCapacity)
|
||||||
|
{
|
||||||
|
_ = Interlocked.Exchange(ref _queue_sync, 0);
|
||||||
|
goto wait_for_spare;
|
||||||
|
}
|
||||||
|
|
||||||
WaitForSpare();
|
|
||||||
|
|
||||||
_consume++;
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
unchecked
|
unchecked
|
||||||
{
|
{
|
||||||
byte idx = (byte)_consume;
|
byte idx = (byte)_consume;
|
||||||
|
_consume++;
|
||||||
_ring[idx] = predictor;
|
_ring[idx] = predictor;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -90,12 +103,28 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
internal bool TryGetCurrentPredictor(
|
||||||
|
out IL4Predictor predictor
|
||||||
|
)
|
||||||
|
{
|
||||||
|
int cur = _predict;
|
||||||
|
if (cur >= _enter)
|
||||||
|
{
|
||||||
|
predictor = default!;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
unchecked
|
||||||
|
{
|
||||||
|
predictor = _ring[(byte)cur];
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private void WaitForSpare()
|
private void WaitForSpare()
|
||||||
{
|
{
|
||||||
int spin_count = 0;
|
int spin_count = 0;
|
||||||
|
|
||||||
// wait until the queue has space
|
while (_enter - _consume >= PredictorRingCapacity)
|
||||||
while (_enter - _consume >= 256)
|
|
||||||
{
|
{
|
||||||
switch (spin_count)
|
switch (spin_count)
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -39,8 +39,9 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
public struct AddressSource : IEquatable<AddressSource>
|
public struct AddressSource : IEquatable<AddressSource>
|
||||||
{
|
{
|
||||||
|
|
||||||
|
public uint Description;
|
||||||
|
public uint Size;
|
||||||
public ulong Address;
|
public ulong Address;
|
||||||
public ulong Description;
|
|
||||||
|
|
||||||
public readonly bool Equals(
|
public readonly bool Equals(
|
||||||
AddressSource other
|
AddressSource other
|
||||||
@@ -102,10 +103,10 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
private readonly SwissBucket* _buckets = (SwissBucket*)(buffer + IdealCount);
|
private readonly SwissBucket* _buckets = (SwissBucket*)(buffer + IdealCount);
|
||||||
private readonly Vector128<byte>* _control_bytes = (Vector128<byte>*)buffer;
|
private readonly Vector128<byte>* _control_bytes = (Vector128<byte>*)buffer;
|
||||||
|
private bool _group_loaded;
|
||||||
private int _current_group;
|
private int _current_group;
|
||||||
private int _current_mask;
|
private int _current_mask;
|
||||||
private int _current_offset;
|
private int _current_offset;
|
||||||
private bool _group_loaded;
|
|
||||||
|
|
||||||
public (AddressSource key, ulong value) Current
|
public (AddressSource key, ulong value) Current
|
||||||
{
|
{
|
||||||
@@ -233,8 +234,8 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
while (e.MoveNext())
|
while (e.MoveNext())
|
||||||
{
|
{
|
||||||
(AddressSource key, ulong value) = e.Current;
|
(AddressSource key, ulong value) = e.Current;
|
||||||
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count))
|
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value,
|
||||||
{
|
ref _delete_count)) {
|
||||||
throw new InvalidOperationException("The Swiss table could not fit all active entries during rehash.");
|
throw new InvalidOperationException("The Swiss table could not fit all active entries during rehash.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -261,7 +262,8 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
if (Sse42.IsSupported)
|
if (Sse42.IsSupported)
|
||||||
{
|
{
|
||||||
return SwissWriterX86.TryRemove(_buffer_write, key, out value, ref _delete_count);
|
return SwissWriterX86.TryRemove(_buffer_write, key, out value,
|
||||||
|
ref _delete_count);
|
||||||
} else
|
} else
|
||||||
{
|
{
|
||||||
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
|
||||||
|
|||||||
@@ -5,27 +5,28 @@
|
|||||||
// SPDX-License-Identifier: MIT
|
// SPDX-License-Identifier: MIT
|
||||||
// -----------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------
|
||||||
|
|
||||||
using Nitload.Common.HighPerformance.ProcessorAbstraction;
|
|
||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Numerics;
|
using System.Numerics;
|
||||||
using System.Runtime.InteropServices;
|
|
||||||
using System.Text;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace Nitload.Common.HighPerformance.Memory
|
namespace Nitload.Common.HighPerformance.Memory
|
||||||
{
|
{
|
||||||
public partial class L4
|
public partial class L4
|
||||||
{
|
{
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// L4-specific TLSF allocator whose metadata nodes also serve as cache-line headers.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Payload occupies the first arena region; a fixed metadata slab follows it and is
|
||||||
|
/// carved backward from its aligned high end. Handle 0 is the sentinel, and positive
|
||||||
|
/// handles are metadata slot indices. This layout is intentionally coupled to L4.
|
||||||
|
/// </remarks>
|
||||||
private unsafe struct TLSF
|
private unsafe struct TLSF
|
||||||
{
|
{
|
||||||
|
|
||||||
public const int MaxCacheLineCount = 8192;
|
public const int MaxCacheLineCount = 8192;
|
||||||
|
|
||||||
public static readonly nuint RequestedAllocation =
|
public static readonly nuint RequestedAllocation =
|
||||||
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
|
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf + CacheLineMetadata.SizeOf - 1u;
|
||||||
private readonly CacheLineMetadata* _spareMemory;
|
private readonly CacheLineMetadata* _spareMemory;
|
||||||
private readonly int _align;
|
private readonly int _align;
|
||||||
private readonly int _maxLevel;
|
private readonly int _maxLevel;
|
||||||
@@ -52,12 +53,16 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
|
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||||
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
|
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
|
||||||
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
|
|
||||||
|
// Metadata occupies the reservation after payload: handle 0 is the sentinel, and handles 1..MaxCacheLineCount are carved backward from the aligned high end until the slab is full.
|
||||||
|
nuint metadataRegionEnd = (nuint)externalPointer + byteSize + RequestedAllocation;
|
||||||
|
nuint alignedMetadataEnd = metadataRegionEnd & ~((nuint)CacheLineMetadata.SizeOf - 1u);
|
||||||
|
_spareMemory = (CacheLineMetadata*)(alignedMetadataEnd - CacheLineMetadata.SizeOf);
|
||||||
_metadataEnd = 0;
|
_metadataEnd = 0;
|
||||||
|
|
||||||
// init spare memory node
|
// init spare memory node
|
||||||
|
|
||||||
_spareMemory->PositionMask = 0;
|
_spareMemory->Position = _basePtr;
|
||||||
_spareMemory->SizeMask = _totalSize;
|
_spareMemory->SizeMask = _totalSize;
|
||||||
_spareMemory->NextLevel = 0;
|
_spareMemory->NextLevel = 0;
|
||||||
_spareMemory->NextNear = 0;
|
_spareMemory->NextNear = 0;
|
||||||
@@ -66,7 +71,15 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
}
|
}
|
||||||
|
|
||||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
private int GetLevel(
|
internal readonly CacheLineMetadata* MetadataAt(
|
||||||
|
int handle
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return _spareMemory - handle;
|
||||||
|
}
|
||||||
|
|
||||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||||
|
private readonly int GetLevel(
|
||||||
uint size
|
uint size
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
@@ -93,11 +106,11 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
// remove from position chain
|
// remove from position chain
|
||||||
short prevNearIdx = node->PreviousNear;
|
short prevNearIdx = node->PreviousNear;
|
||||||
short nextNearIdx = node->NextNear;
|
short nextNearIdx = node->NextNear;
|
||||||
CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
|
CacheLineMetadata* prevNear = MetadataAt(prevNearIdx);
|
||||||
prevNear->NextNear = nextNearIdx;
|
prevNear->NextNear = nextNearIdx;
|
||||||
if (nextNearIdx != 0)
|
if (nextNearIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
|
CacheLineMetadata* nextNear = MetadataAt(nextNearIdx);
|
||||||
nextNear->PreviousNear = prevNearIdx;
|
nextNear->PreviousNear = prevNearIdx;
|
||||||
} else
|
} else
|
||||||
{
|
{
|
||||||
@@ -106,7 +119,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
node->PreviousNear = 0;
|
node->PreviousNear = 0;
|
||||||
node->NextNear = 0;
|
node->NextNear = 0;
|
||||||
|
|
||||||
node->PositionMask = 0;
|
node->Position = _basePtr;
|
||||||
node->SizeMask = 0;
|
node->SizeMask = 0;
|
||||||
node->State = AllocationState.SpareOrCollect;
|
node->State = AllocationState.SpareOrCollect;
|
||||||
node->PreviousLevel = 0;
|
node->PreviousLevel = 0;
|
||||||
@@ -114,7 +127,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
short nextIdx = _spareMemory->NextLevel;
|
short nextIdx = _spareMemory->NextLevel;
|
||||||
if (nextIdx != 0)
|
if (nextIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
CacheLineMetadata* next = MetadataAt(nextIdx);
|
||||||
next->PreviousLevel = nodeIdx;
|
next->PreviousLevel = nodeIdx;
|
||||||
}
|
}
|
||||||
node->NextLevel = nextIdx;
|
node->NextLevel = nextIdx;
|
||||||
@@ -145,12 +158,12 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
_bucket[currentLevel] = currentNextLevelIdx;
|
_bucket[currentLevel] = currentNextLevelIdx;
|
||||||
} else if (currentPrevLevelIdx != 0)
|
} else if (currentPrevLevelIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
CacheLineMetadata* prevLevel = MetadataAt(currentPrevLevelIdx);
|
||||||
prevLevel->NextLevel = currentNextLevelIdx;
|
prevLevel->NextLevel = currentNextLevelIdx;
|
||||||
}
|
}
|
||||||
if (currentNextLevelIdx != 0)
|
if (currentNextLevelIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
CacheLineMetadata* nextLevel = MetadataAt(currentNextLevelIdx);
|
||||||
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
||||||
}
|
}
|
||||||
node->PreviousLevel = 0;
|
node->PreviousLevel = 0;
|
||||||
@@ -203,11 +216,11 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
meta = null;
|
meta = null;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
meta = &_spareMemory[idx];
|
meta = MetadataAt(idx);
|
||||||
short nextSpareIdx = meta->NextLevel;
|
short nextSpareIdx = meta->NextLevel;
|
||||||
if (nextSpareIdx != 0)
|
if (nextSpareIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
|
CacheLineMetadata* nextSpare = MetadataAt(nextSpareIdx);
|
||||||
nextSpare->PreviousLevel = 0;
|
nextSpare->PreviousLevel = 0;
|
||||||
}
|
}
|
||||||
_spareMemory->NextLevel = nextSpareIdx;
|
_spareMemory->NextLevel = nextSpareIdx;
|
||||||
@@ -215,18 +228,19 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
if (!TryGetNewMetadata(out meta)) {
|
if (!TryGetNewMetadata(out meta)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
idx = (short)(_spareMemory - meta);
|
||||||
spareSize -= size;
|
spareSize -= size;
|
||||||
meta->SizeMask = size;
|
meta->SizeMask = size;
|
||||||
meta->PositionMask = _spareMemory->PositionMask;
|
meta->Position = _spareMemory->Position;
|
||||||
meta->State = AllocationState.Free;
|
meta->State = AllocationState.Free;
|
||||||
meta->PreviousLevel = 0;
|
meta->PreviousLevel = 0;
|
||||||
meta->NextLevel = 0;
|
meta->NextLevel = 0;
|
||||||
_spareMemory->SizeMask = spareSize;
|
_spareMemory->SizeMask = spareSize;
|
||||||
_spareMemory->PositionMask += size;
|
_spareMemory->Position += size;
|
||||||
|
|
||||||
// insert the new chunk to position chain
|
// insert the new chunk to position chain
|
||||||
short nearIdx = _spareMemory->PreviousNear;
|
short nearIdx = _spareMemory->PreviousNear;
|
||||||
CacheLineMetadata* near = &_spareMemory[nearIdx];
|
CacheLineMetadata* near = MetadataAt(nearIdx);
|
||||||
near->NextNear = idx;
|
near->NextNear = idx;
|
||||||
meta->PreviousNear = nearIdx;
|
meta->PreviousNear = nearIdx;
|
||||||
_spareMemory->PreviousNear = idx;
|
_spareMemory->PreviousNear = idx;
|
||||||
@@ -245,7 +259,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
_metadataEnd++;
|
_metadataEnd++;
|
||||||
meta = &_spareMemory[_metadataEnd];
|
meta = MetadataAt(_metadataEnd);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,7 +275,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
int idx = _bucket[level];
|
int idx = _bucket[level];
|
||||||
while (idx != 0)
|
while (idx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
CacheLineMetadata* candidate = MetadataAt(idx);
|
||||||
if (candidate->SizeMask >= minimumSize)
|
if (candidate->SizeMask >= minimumSize)
|
||||||
{
|
{
|
||||||
RemoveFromBucket(level, candidate, idx);
|
RemoveFromBucket(level, candidate, idx);
|
||||||
@@ -276,7 +290,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
int idx = _bucket[i];
|
int idx = _bucket[i];
|
||||||
if (idx != 0)
|
if (idx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
CacheLineMetadata* candidate = MetadataAt(idx);
|
||||||
RemoveFromBucketHead(i, candidate, idx);
|
RemoveFromBucketHead(i, candidate, idx);
|
||||||
entry = candidate;
|
entry = candidate;
|
||||||
return true;
|
return true;
|
||||||
@@ -313,7 +327,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||||
{
|
{
|
||||||
short newEntryIdx = (short)(_spareMemory - newEntry);
|
short newEntryIdx = (short)(_spareMemory - newEntry);
|
||||||
newEntry->PositionMask = entry->PositionMask + allocSize;
|
newEntry->Position = entry->Position + allocSize;
|
||||||
newEntry->SizeMask = remain;
|
newEntry->SizeMask = remain;
|
||||||
newEntry->State = AllocationState.Free;
|
newEntry->State = AllocationState.Free;
|
||||||
int newLevel = GetLevel(remain);
|
int newLevel = GetLevel(remain);
|
||||||
@@ -321,7 +335,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
if (bucketHeadIdx != 0)
|
if (bucketHeadIdx != 0)
|
||||||
{
|
{
|
||||||
// if there is already an entry in the bucket, link it to the new entry
|
// if there is already an entry in the bucket, link it to the new entry
|
||||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
|
||||||
bucketHead->PreviousLevel = newEntryIdx;
|
bucketHead->PreviousLevel = newEntryIdx;
|
||||||
}
|
}
|
||||||
newEntry->NextLevel = bucketHeadIdx;
|
newEntry->NextLevel = bucketHeadIdx;
|
||||||
@@ -331,14 +345,14 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
// push to position chain
|
// push to position chain
|
||||||
short afterIdx = entry->NextNear;
|
short afterIdx = entry->NextNear;
|
||||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
CacheLineMetadata* after = MetadataAt(afterIdx);
|
||||||
newEntry->NextNear = afterIdx;
|
newEntry->NextNear = afterIdx;
|
||||||
after->PreviousNear = newEntryIdx;
|
after->PreviousNear = newEntryIdx;
|
||||||
entry->NextNear = newEntryIdx;
|
entry->NextNear = newEntryIdx;
|
||||||
newEntry->PreviousNear = entryIdx;
|
newEntry->PreviousNear = entryIdx;
|
||||||
}
|
}
|
||||||
entry->State = AllocationState.Alloc;
|
entry->State = AllocationState.Alloc;
|
||||||
if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
|
if (entry->Position - (ulong)_basePtr > _totalSize) {
|
||||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||||
}
|
}
|
||||||
return entryIdx;
|
return entryIdx;
|
||||||
@@ -353,7 +367,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
CacheLineMetadata* current = &_spareMemory[currentIdx];
|
CacheLineMetadata* current = MetadataAt(currentIdx);
|
||||||
if (current->State == AllocationState.Free)
|
if (current->State == AllocationState.Free)
|
||||||
{
|
{
|
||||||
// scan next
|
// scan next
|
||||||
@@ -361,7 +375,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
int nextIdx = current->NextNear;
|
int nextIdx = current->NextNear;
|
||||||
while (nextIdx != 0)
|
while (nextIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
CacheLineMetadata* next = MetadataAt(nextIdx);
|
||||||
if (next->State != AllocationState.Free)
|
if (next->State != AllocationState.Free)
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
@@ -378,7 +392,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
short bucketHeadIdx = _bucket[newLevel];
|
short bucketHeadIdx = _bucket[newLevel];
|
||||||
if (bucketHeadIdx != 0)
|
if (bucketHeadIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
|
||||||
bucketHead->PreviousLevel = currentIdx;
|
bucketHead->PreviousLevel = currentIdx;
|
||||||
}
|
}
|
||||||
current->NextLevel = bucketHeadIdx;
|
current->NextLevel = bucketHeadIdx;
|
||||||
@@ -395,7 +409,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
int collectScanWindow = 4
|
int collectScanWindow = 4
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* head = &_spareMemory[handle];
|
CacheLineMetadata* head = MetadataAt(handle);
|
||||||
short headIdx = (short)(_spareMemory - head);
|
short headIdx = (short)(_spareMemory - head);
|
||||||
uint size = head->SizeMask;
|
uint size = head->SizeMask;
|
||||||
for (int i = 0; i < collectScanWindow; i++)
|
for (int i = 0; i < collectScanWindow; i++)
|
||||||
@@ -406,13 +420,13 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
CacheLineMetadata* prev = &_spareMemory[forwardIdx];
|
CacheLineMetadata* prev = MetadataAt(forwardIdx);
|
||||||
if (prev->State != AllocationState.Free)
|
if (prev->State != AllocationState.Free)
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
size += prev->SizeMask;
|
size += prev->SizeMask;
|
||||||
head->PositionMask = prev->PositionMask;
|
head->Position = prev->Position;
|
||||||
RecycleNode(prev);
|
RecycleNode(prev);
|
||||||
}
|
}
|
||||||
for (int i = 0; i < collectScanWindow; i++)
|
for (int i = 0; i < collectScanWindow; i++)
|
||||||
@@ -423,7 +437,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
CacheLineMetadata* prev = &_spareMemory[nextIdx];
|
CacheLineMetadata* prev = MetadataAt(nextIdx);
|
||||||
if (prev->State != AllocationState.Free)
|
if (prev->State != AllocationState.Free)
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
@@ -441,7 +455,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
head->NextLevel = 0;
|
head->NextLevel = 0;
|
||||||
if (bucketHeadIdx != 0)
|
if (bucketHeadIdx != 0)
|
||||||
{
|
{
|
||||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
|
||||||
bucketHead->PreviousLevel = headIdx;
|
bucketHead->PreviousLevel = headIdx;
|
||||||
head->NextLevel = bucketHeadIdx;
|
head->NextLevel = bucketHeadIdx;
|
||||||
}
|
}
|
||||||
@@ -479,7 +493,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||||
{
|
{
|
||||||
short newEntryIdx = (short)(_spareMemory - newEntry);
|
short newEntryIdx = (short)(_spareMemory - newEntry);
|
||||||
newEntry->PositionMask = entry->PositionMask + allocSize;
|
newEntry->Position = entry->Position + allocSize;
|
||||||
newEntry->SizeMask = remain;
|
newEntry->SizeMask = remain;
|
||||||
newEntry->State = AllocationState.Free;
|
newEntry->State = AllocationState.Free;
|
||||||
int newLevel = GetLevel(remain);
|
int newLevel = GetLevel(remain);
|
||||||
@@ -487,7 +501,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
if (bucketHeadIdx != 0)
|
if (bucketHeadIdx != 0)
|
||||||
{
|
{
|
||||||
// if there is already an entry in the bucket, link it to the new entry
|
// if there is already an entry in the bucket, link it to the new entry
|
||||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
|
||||||
newEntry->NextLevel = bucketHeadIdx;
|
newEntry->NextLevel = bucketHeadIdx;
|
||||||
bucketHead->PreviousLevel = newEntryIdx;
|
bucketHead->PreviousLevel = newEntryIdx;
|
||||||
} else
|
} else
|
||||||
@@ -499,14 +513,14 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
|
|
||||||
// push to position chain
|
// push to position chain
|
||||||
short afterIdx = entry->NextNear;
|
short afterIdx = entry->NextNear;
|
||||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
CacheLineMetadata* after = MetadataAt(afterIdx);
|
||||||
newEntry->NextNear = afterIdx;
|
newEntry->NextNear = afterIdx;
|
||||||
after->PreviousNear = newEntryIdx;
|
after->PreviousNear = newEntryIdx;
|
||||||
entry->NextNear = newEntryIdx;
|
entry->NextNear = newEntryIdx;
|
||||||
newEntry->PreviousNear = entryIdx;
|
newEntry->PreviousNear = entryIdx;
|
||||||
}
|
}
|
||||||
entry->State = AllocationState.Alloc;
|
entry->State = AllocationState.Alloc;
|
||||||
if (entry->PositionMask - (ulong)_basePtr > _totalSize)
|
if (entry->Position - (ulong)_basePtr > _totalSize)
|
||||||
{
|
{
|
||||||
h = default;
|
h = default;
|
||||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||||
|
|||||||
@@ -13,10 +13,12 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
|
|
||||||
private const nuint PageSize = 4096;
|
private const nuint PageSize = 4096;
|
||||||
|
private const int PredictorRingCapacity = 256;
|
||||||
|
|
||||||
private bool _disposed;
|
private bool _disposed;
|
||||||
private readonly int _bindedNuma;
|
private readonly int _bindedNuma;
|
||||||
private readonly nuint _baseAddress;
|
private readonly nuint _baseAddress;
|
||||||
|
private readonly IL4Predictor[] _ring;
|
||||||
private SwissTable _addressMap;
|
private SwissTable _addressMap;
|
||||||
private TLSF _bufferAllocator;
|
private TLSF _bufferAllocator;
|
||||||
|
|
||||||
@@ -29,6 +31,7 @@ namespace Nitload.Common.HighPerformance.Memory
|
|||||||
{
|
{
|
||||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
|
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
|
||||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
|
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
|
||||||
|
_ring = new IL4Predictor[PredictorRingCapacity];
|
||||||
_bindedNuma = bindedNuma;
|
_bindedNuma = bindedNuma;
|
||||||
_predict_slow_path_count = GetPredictSlowPathCount(bindedNuma);
|
_predict_slow_path_count = GetPredictSlowPathCount(bindedNuma);
|
||||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
|
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
|
||||||
|
|||||||
Reference in New Issue
Block a user