youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

predict method of l4

Browse Source
This commit is contained in:
Youmu Liu
2026-10-19 18:38:43 +08:00
parent d1efb15e3f
commit e349bcc4dd
7 changed files with 195 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs
+39 -3
View File
@@ -6,6 +6,7 @@
// -----------------------------------------------------------------------------
using System.Runtime.InteropServices;
using System.Xml.Serialization;
namespace Nitload.Common.HighPerformance.Memory
{
@@ -27,10 +28,38 @@ namespace Nitload.Common.HighPerformance.Memory
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(16)] internal AllocationState State;
[FieldOffset(31)] internal AllocationState State;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
/// <summary>
/// only available in L4.Predict()
/// </summary>
public void CheckCount()
{
uint count = Volatile.Read(ref ConsumeCount);
if (ushort.MaxValue < count)
{
uint delta = unchecked(0u - count);
_ = Interlocked.Add(ref ConsumeCount, delta);
_ = Interlocked.Add(ref PredictCount, delta);
}
}
public void IncreaseConsumeCount()
{
unchecked {
_ = Interlocked.Increment(ref ConsumeCount);
}
}
public void IncreasePredictCount()
{
unchecked {
_ = Interlocked.Increment(ref PredictCount);
}
}
#region tlsf linked list
[FieldOffset(0)] public short NextLevel;
@@ -42,8 +71,15 @@ namespace Nitload.Common.HighPerformance.Memory
#region buffer
[FieldOffset(8)] public uint PositionMask;
[FieldOffset(12)] public uint SizeMask;
[FieldOffset(8)] public ulong Position;
[FieldOffset(16)] public uint SizeMask;
#endregion
#region count
[FieldOffset(20)] public uint PredictCount;
[FieldOffset(24)] public uint ConsumeCount;
#endregion
}
src/Nitload.Common/HighPerformance/Memory/L4.Map.cs
+2 -2
View File
@@ -72,8 +72,8 @@ namespace Nitload.Common.HighPerformance.Memory
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
// process metadata, unfinished
uint offset = metadata->PositionMask;
nuint address = /*base ptr + */ offset;
ulong offset = metadata->Position;
nuint address = /*base ptr + */ (nuint)offset;
_ = Interlocked.Decrement(ref _predict_count);
return new Handle(address, (nuint)handle);
src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs
+50 -1
View File
@@ -18,8 +18,57 @@ namespace Nitload.Common.HighPerformance.Memory
internal partial int Predict()
{
throw new NotImplementedException();
if (!TryGetCurrentPredictor(out IL4Predictor? p)) {
return 0;
}
int total_count = 0, count;
Span<L4.AddressSource> as_sp = stackalloc L4.AddressSource[4];
Predict:
while ((count = p.PredictMission(in as_sp)) > 0)
{
for (int i = 0; i < count; i++)
{
if (_addressMap.TryGet(as_sp[i], out ulong map_handle_adr))
{
L4.CacheLineMetadata* map_handle = (L4.CacheLineMetadata*)map_handle_adr;
map_handle->IncreasePredictCount();
map_handle->CheckCount();
} else
{
uint size = as_sp[i].Size;
int handle_idx = _bufferAllocator.Alloc(size);
L4.CacheLineMetadata* handle = _bufferAllocator.MetadataAt(handle_idx);
if (!_addressMap.TryAddOrSet(as_sp[i], (ulong)handle)) {
throw new InvalidOperationException();
}
}
}
total_count += count;
if (total_count >= 2048) {
return total_count;
}
}
if (MoveNextPredictor())
{
_ = TryGetCurrentPredictor(out p);
goto Predict;
}
return total_count;
}
}
}
/*
goto 本身没问题，但当前控制流有几个需要先确认的点：
高：换 predictor 的分支看起来反了。 MoveNextPredictor() 在 _predict 到达 _enter 时返回 true。因此 L4.Predict.cs:51 现在会在队列耗尽时再次取 predictor；TryGetCurrentPredictor 失败后仍 goto Predict，接着会用空的 p 调 PredictMission。反过来，有下一个 predictor 时它会提前返回，不再处理后续 predictor。
高：新分配的 metadata 没有增加预测计数。 现有映射分支会调用 IncreasePredictCount()，但新分配分支只分配并写入 Swiss。若该计数表示未消费预测数，新条目也需要相同的计数处理；还要确认 TLSF 重用 metadata 时会重置旧计数。
高：同一轮内重复的 AddressSource 可能重复分配。 SwissTable.TryGet 只查 published buffer，而新条目先写入 inactive buffer。后续再次预测相同 key 时会查不到，重新分配 metadata，再由 TryAddOrSet 覆盖先前 value，旧分配就失去映射。
中：需要校验 PredictMission 的返回值范围。 Span 容量是 4；count > 4 会让 as_sp[i] 越界，负值则被当作本轮结束。建议契约明确要求 0..4，并在 L4 边界验证。
本轮未改代码。L4.Predict.cs 的文件诊断通过，但尚未运行完整构建或运行时测试。
*/
src/Nitload.Common/HighPerformance/Memory/L4.PredictorQueue.cs
+36 -7
View File
@@ -16,7 +16,6 @@ namespace Nitload.Common.HighPerformance.Memory
public partial class L4
{
private readonly IL4Predictor[] _ring = new IL4Predictor[256];
private volatile int _consume;
private volatile int _enter;
private volatile int _predict;
@@ -45,7 +44,8 @@ namespace Nitload.Common.HighPerformance.Memory
if (ReferenceEquals(predictor, _ring[idx]))
{
int end = _predict;
for (int move = i; move < end - 1; move++) {
for (int move = i; move < end - 1; move++)
{
_ring[(byte)move] = _ring[(byte)(move + 1)];
}
_ring[(byte)(end - 1)] = default!;
@@ -61,6 +61,12 @@ namespace Nitload.Common.HighPerformance.Memory
}
}
internal bool MoveNextPredictor()
{
int cur = Interlocked.Increment(ref _predict);
return cur == _enter;
}
internal void RegistPredictor(
IL4Predictor predictor
)
@@ -69,18 +75,25 @@ namespace Nitload.Common.HighPerformance.Memory
return;
}
SpinWait sp = new();
wait_for_spare:
WaitForSpare();
while (Interlocked.CompareExchange(ref _queue_sync, 1, 0) != 0) {
sp.SpinOnce();
}
if (_enter - _consume >= PredictorRingCapacity)
{
_ = Interlocked.Exchange(ref _queue_sync, 0);
goto wait_for_spare;
}
WaitForSpare();
_consume++;
try
{
unchecked
{
byte idx = (byte)_consume;
_consume++;
_ring[idx] = predictor;
}
}
@@ -90,12 +103,28 @@ namespace Nitload.Common.HighPerformance.Memory
}
}
internal bool TryGetCurrentPredictor(
out IL4Predictor predictor
)
{
int cur = _predict;
if (cur >= _enter)
{
predictor = default!;
return false;
}
unchecked
{
predictor = _ring[(byte)cur];
return true;
}
}
private void WaitForSpare()
{
int spin_count = 0;
// wait until the queue has space
while (_enter - _consume >= 256)
while (_enter - _consume >= PredictorRingCapacity)
{
switch (spin_count)
{
src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs
+9 -7
View File
@@ -39,8 +39,9 @@ namespace Nitload.Common.HighPerformance.Memory
public struct AddressSource : IEquatable<AddressSource>
{
public uint Description;
public uint Size;
public ulong Address;
public ulong Description;
public readonly bool Equals(
AddressSource other
@@ -102,10 +103,10 @@ namespace Nitload.Common.HighPerformance.Memory
private readonly SwissBucket* _buckets = (SwissBucket*)(buffer + IdealCount);
private readonly Vector128<byte>* _control_bytes = (Vector128<byte>*)buffer;
private bool _group_loaded;
private int _current_group;
private int _current_mask;
private int _current_offset;
private bool _group_loaded;
public (AddressSource key, ulong value) Current
{
@@ -190,8 +191,8 @@ namespace Nitload.Common.HighPerformance.Memory
/// Writes to the inactive buffer. The caller must serialize all writes.
/// </summary>
public bool TryAddOrSet(
AddressSource key,
ulong value
AddressSource key,
ulong value
)
{
if (Sse42.IsSupported) {
@@ -233,8 +234,8 @@ namespace Nitload.Common.HighPerformance.Memory
while (e.MoveNext())
{
(AddressSource key, ulong value) = e.Current;
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value, ref _delete_count))
{
if (!SwissWriterX86.TryAddOrSet(_buffer_write, key, value,
ref _delete_count)) {
throw new InvalidOperationException("The Swiss table could not fit all active entries during rehash.");
}
}
@@ -261,7 +262,8 @@ namespace Nitload.Common.HighPerformance.Memory
{
if (Sse42.IsSupported)
{
return SwissWriterX86.TryRemove(_buffer_write, key, out value, ref _delete_count);
return SwissWriterX86.TryRemove(_buffer_write, key, out value,
ref _delete_count);
} else
{
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs
+56 -42
View File
@@ -5,27 +5,28 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
/// <summary>
/// L4-specific TLSF allocator whose metadata nodes also serve as cache-line headers.
/// </summary>
/// <remarks>
/// Payload occupies the first arena region; a fixed metadata slab follows it and is
/// carved backward from its aligned high end. Handle 0 is the sentinel, and positive
/// handles are metadata slot indices. This layout is intentionally coupled to L4.
/// </remarks>
private unsafe struct TLSF
{
public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation =
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf + CacheLineMetadata.SizeOf - 1u;
private readonly CacheLineMetadata* _spareMemory;
private readonly int _align;
private readonly int _maxLevel;
@@ -52,12 +53,16 @@ namespace Nitload.Common.HighPerformance.Memory
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
// Metadata occupies the reservation after payload: handle 0 is the sentinel, and handles 1..MaxCacheLineCount are carved backward from the aligned high end until the slab is full.
nuint metadataRegionEnd = (nuint)externalPointer + byteSize + RequestedAllocation;
nuint alignedMetadataEnd = metadataRegionEnd & ~((nuint)CacheLineMetadata.SizeOf - 1u);
_spareMemory = (CacheLineMetadata*)(alignedMetadataEnd - CacheLineMetadata.SizeOf);
_metadataEnd = 0;
// init spare memory node
_spareMemory->PositionMask = 0;
_spareMemory->Position = _basePtr;
_spareMemory->SizeMask = _totalSize;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
@@ -66,8 +71,16 @@ namespace Nitload.Common.HighPerformance.Memory
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(
uint size
internal readonly CacheLineMetadata* MetadataAt(
int handle
)
{
return _spareMemory - handle;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private readonly int GetLevel(
uint size
)
{
int origin = 25 - BitOperations.LeadingZeroCount(size);
@@ -93,11 +106,11 @@ namespace Nitload.Common.HighPerformance.Memory
// remove from position chain
short prevNearIdx = node->PreviousNear;
short nextNearIdx = node->NextNear;
CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
CacheLineMetadata* prevNear = MetadataAt(prevNearIdx);
prevNear->NextNear = nextNearIdx;
if (nextNearIdx != 0)
{
CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
CacheLineMetadata* nextNear = MetadataAt(nextNearIdx);
nextNear->PreviousNear = prevNearIdx;
} else
{
@@ -106,7 +119,7 @@ namespace Nitload.Common.HighPerformance.Memory
node->PreviousNear = 0;
node->NextNear = 0;
node->PositionMask = 0;
node->Position = _basePtr;
node->SizeMask = 0;
node->State = AllocationState.SpareOrCollect;
node->PreviousLevel = 0;
@@ -114,7 +127,7 @@ namespace Nitload.Common.HighPerformance.Memory
short nextIdx = _spareMemory->NextLevel;
if (nextIdx != 0)
{
CacheLineMetadata* next = &_spareMemory[nextIdx];
CacheLineMetadata* next = MetadataAt(nextIdx);
next->PreviousLevel = nodeIdx;
}
node->NextLevel = nextIdx;
@@ -145,12 +158,12 @@ namespace Nitload.Common.HighPerformance.Memory
_bucket[currentLevel] = currentNextLevelIdx;
} else if (currentPrevLevelIdx != 0)
{
CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
CacheLineMetadata* prevLevel = MetadataAt(currentPrevLevelIdx);
prevLevel->NextLevel = currentNextLevelIdx;
}
if (currentNextLevelIdx != 0)
{
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
CacheLineMetadata* nextLevel = MetadataAt(currentNextLevelIdx);
nextLevel->PreviousLevel = currentPrevLevelIdx;
}
node->PreviousLevel = 0;
@@ -203,11 +216,11 @@ namespace Nitload.Common.HighPerformance.Memory
meta = null;
return false;
}
meta = &_spareMemory[idx];
meta = MetadataAt(idx);
short nextSpareIdx = meta->NextLevel;
if (nextSpareIdx != 0)
{
CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
CacheLineMetadata* nextSpare = MetadataAt(nextSpareIdx);
nextSpare->PreviousLevel = 0;
}
_spareMemory->NextLevel = nextSpareIdx;
@@ -215,18 +228,19 @@ namespace Nitload.Common.HighPerformance.Memory
if (!TryGetNewMetadata(out meta)) {
return false;
}
idx = (short)(_spareMemory - meta);
spareSize -= size;
meta->SizeMask = size;
meta->PositionMask = _spareMemory->PositionMask;
meta->Position = _spareMemory->Position;
meta->State = AllocationState.Free;
meta->PreviousLevel = 0;
meta->NextLevel = 0;
_spareMemory->SizeMask = spareSize;
_spareMemory->PositionMask += size;
_spareMemory->Position += size;
// insert the new chunk to position chain
short nearIdx = _spareMemory->PreviousNear;
CacheLineMetadata* near = &_spareMemory[nearIdx];
CacheLineMetadata* near = MetadataAt(nearIdx);
near->NextNear = idx;
meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = idx;
@@ -245,7 +259,7 @@ namespace Nitload.Common.HighPerformance.Memory
return false;
}
_metadataEnd++;
meta = &_spareMemory[_metadataEnd];
meta = MetadataAt(_metadataEnd);
return true;
}
@@ -261,7 +275,7 @@ namespace Nitload.Common.HighPerformance.Memory
int idx = _bucket[level];
while (idx != 0)
{
CacheLineMetadata* candidate = &_spareMemory[idx];
CacheLineMetadata* candidate = MetadataAt(idx);
if (candidate->SizeMask >= minimumSize)
{
RemoveFromBucket(level, candidate, idx);
@@ -276,7 +290,7 @@ namespace Nitload.Common.HighPerformance.Memory
int idx = _bucket[i];
if (idx != 0)
{
CacheLineMetadata* candidate = &_spareMemory[idx];
CacheLineMetadata* candidate = MetadataAt(idx);
RemoveFromBucketHead(i, candidate, idx);
entry = candidate;
return true;
@@ -313,7 +327,7 @@ namespace Nitload.Common.HighPerformance.Memory
TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->Position = entry->Position + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
@@ -321,7 +335,7 @@ namespace Nitload.Common.HighPerformance.Memory
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
bucketHead->PreviousLevel = newEntryIdx;
}
newEntry->NextLevel = bucketHeadIdx;
@@ -331,14 +345,14 @@ namespace Nitload.Common.HighPerformance.Memory
// push to position chain
short afterIdx = entry->NextNear;
CacheLineMetadata* after = &_spareMemory[afterIdx];
CacheLineMetadata* after = MetadataAt(afterIdx);
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
if (entry->Position - (ulong)_basePtr > _totalSize) {
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
return entryIdx;
@@ -353,7 +367,7 @@ namespace Nitload.Common.HighPerformance.Memory
{
break;
}
CacheLineMetadata* current = &_spareMemory[currentIdx];
CacheLineMetadata* current = MetadataAt(currentIdx);
if (current->State == AllocationState.Free)
{
// scan next
@@ -361,7 +375,7 @@ namespace Nitload.Common.HighPerformance.Memory
int nextIdx = current->NextNear;
while (nextIdx != 0)
{
CacheLineMetadata* next = &_spareMemory[nextIdx];
CacheLineMetadata* next = MetadataAt(nextIdx);
if (next->State != AllocationState.Free)
{
break;
@@ -378,7 +392,7 @@ namespace Nitload.Common.HighPerformance.Memory
short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
bucketHead->PreviousLevel = currentIdx;
}
current->NextLevel = bucketHeadIdx;
@@ -395,7 +409,7 @@ namespace Nitload.Common.HighPerformance.Memory
int collectScanWindow = 4
)
{
CacheLineMetadata* head = &_spareMemory[handle];
CacheLineMetadata* head = MetadataAt(handle);
short headIdx = (short)(_spareMemory - head);
uint size = head->SizeMask;
for (int i = 0; i < collectScanWindow; i++)
@@ -406,13 +420,13 @@ namespace Nitload.Common.HighPerformance.Memory
{
break;
}
CacheLineMetadata* prev = &_spareMemory[forwardIdx];
CacheLineMetadata* prev = MetadataAt(forwardIdx);
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->SizeMask;
head->PositionMask = prev->PositionMask;
head->Position = prev->Position;
RecycleNode(prev);
}
for (int i = 0; i < collectScanWindow; i++)
@@ -423,7 +437,7 @@ namespace Nitload.Common.HighPerformance.Memory
{
break;
}
CacheLineMetadata* prev = &_spareMemory[nextIdx];
CacheLineMetadata* prev = MetadataAt(nextIdx);
if (prev->State != AllocationState.Free)
{
break;
@@ -441,7 +455,7 @@ namespace Nitload.Common.HighPerformance.Memory
head->NextLevel = 0;
if (bucketHeadIdx != 0)
{
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
bucketHead->PreviousLevel = headIdx;
head->NextLevel = bucketHeadIdx;
}
@@ -479,7 +493,7 @@ namespace Nitload.Common.HighPerformance.Memory
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->Position = entry->Position + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
@@ -487,7 +501,7 @@ namespace Nitload.Common.HighPerformance.Memory
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
CacheLineMetadata* bucketHead = MetadataAt(bucketHeadIdx);
newEntry->NextLevel = bucketHeadIdx;
bucketHead->PreviousLevel = newEntryIdx;
} else
@@ -499,14 +513,14 @@ namespace Nitload.Common.HighPerformance.Memory
// push to position chain
short afterIdx = entry->NextNear;
CacheLineMetadata* after = &_spareMemory[afterIdx];
CacheLineMetadata* after = MetadataAt(afterIdx);
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->PositionMask - (ulong)_basePtr > _totalSize)
if (entry->Position - (ulong)_basePtr > _totalSize)
{
h = default;
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
src/Nitload.Common/HighPerformance/Memory/L4.cs
+3
View File
@@ -13,10 +13,12 @@ namespace Nitload.Common.HighPerformance.Memory
{
private const nuint PageSize = 4096;
private const int PredictorRingCapacity = 256;
private bool _disposed;
private readonly int _bindedNuma;
private readonly nuint _baseAddress;
private readonly IL4Predictor[] _ring;
private SwissTable _addressMap;
private TLSF _bufferAllocator;
@@ -29,6 +31,7 @@ namespace Nitload.Common.HighPerformance.Memory
{
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_ring = new IL4Predictor[PredictorRingCapacity];
_bindedNuma = bindedNuma;
_predict_slow_path_count = GetPredictSlowPathCount(bindedNuma);
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));