youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

L4 is now on new algo, using new eviction arrays, using union of hlsf and cache line head.

Browse Source
This commit is contained in:
Youmu Liu
2026-04-23 17:39:38 +08:00
parent 5252be9436
commit 1a6ce24a27
5 changed files with 117 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+12 -4
View File
@@ -35,10 +35,18 @@ namespace StgSharp.HighPerformance.Memory
{
[InlineArray(CacheLineCount)]
private struct EvictionArray
private struct PredictionArray
{
private byte pprv;
private short PredictionCount;
}
[InlineArray(CacheLineCount)]
private struct MapArray
{
private short MapCount;
}
@@ -51,8 +59,8 @@ namespace StgSharp.HighPerformance.Memory
#region reference counting and state information
[FieldOffset(52)] internal int RefCount;
[FieldOffset(56)] internal int RemainingPredictionCount;
[FieldOffset(60)] internal int Id;
[FieldOffset(60)] private readonly int Reverse;
[FieldOffset(56)] internal int Id;
#endregion
StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+27 -44
View File
@@ -34,7 +34,6 @@ using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
using HlsfEntry = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator.Entry;
namespace StgSharp.HighPerformance.Memory
@@ -42,12 +41,6 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe void AgeAll()
{
_intrinsic.AgeAll(_eviction);
}
/// <summary>
///
/// </summary>
@@ -62,41 +55,8 @@ namespace StgSharp.HighPerformance.Memory
int count = 1
)
{
int remain = count;
Vector128<byte> three = Vector128.Create((byte)0x03);
byte* p = (byte*)_eviction;
_predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = _predictors.AsSpan();
try
{
for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16)
{
int evictMask = _intrinsic.ScanEvict(p + i, three);
while ((evictMask != 0) && (remain > 0))
{
int bitPos = BitOperations.TrailingZeroCount(evictMask);
int index = bitPos + i;
CacheLineHead* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
int id = head->Predictor;
IL4Predict predict = pSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
RecycleLine(index);
remain--;
}
evictMask &= evictMask - 1;
}
}
return remain;
}
finally
{
_predictors.ReleaseSpanOperation();
}
count = (count < 0) ? 0 : count;
return _intrinsic.ScanEvict((short*)_predictCount, (short*)_mapCount, count, this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -108,6 +68,29 @@ namespace StgSharp.HighPerformance.Memory
_bufferAllocator.Free(entry);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index,
Span<IL4Predict> predictorSpan
)
{
CacheLineHead* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
int id = head->Predictor;
IL4Predict predict = predictorSpan[id] ??
throw new L4PredictUnregisteredException(id);
predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
RecycleLine(index);
(*_mapCount)[index] = 0;
(*_predictCount)[index] = short.MaxValue;
return true;
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryGetEmptyLine(
nuint size,
@@ -120,8 +103,8 @@ namespace StgSharp.HighPerformance.Memory
index = 0;
return false;
}
Hlsf.Entry* entry = handle.EntryHandle;
index = (int)(entry - ((HlsfEntry*)_entryManager._basePtr));
CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
index = entry->Id;
return true;
}
StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs
+41 -29
View File
@@ -25,6 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -44,15 +45,12 @@ namespace StgSharp.HighPerformance.Memory
static IIntrinsicKernel Instance { get; } = Create();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AgeAll(
EvictionArray* eviction
);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ScanEvict(
byte* vPtr,
Vector128<byte> three
short* pPtr,
short* mPtr,
int count,
L4 owner
);
private static IIntrinsicKernel Create()
@@ -68,35 +66,49 @@ namespace StgSharp.HighPerformance.Memory
}
private struct SseKernel : IIntrinsicKernel
private readonly struct SseKernel : IIntrinsicKernel
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void AgeAll(
EvictionArray* eviction
public readonly int ScanEvict(
short* pPtr,
short* mPtr,
int count,
L4 owner
)
{
Vector128<byte> one = Vector128.Create((byte)0x01);
Vector128<byte> max = Vector128.Create((byte)0x03);
byte* p = (byte*)eviction;
for (int i = 0; i < CacheLineCount; i += 16)
int remain = count;
ConcurrentFlexibleArray<IL4Predict> predictors = owner._predictors;
predictors.RequestSpanOperation();
Span<IL4Predict> pSpan = predictors.AsSpan();
try
{
Vector128<byte> v = Sse2.LoadVector128(p + i);
v = Sse2.AddSaturate(v, one);
v = Sse2.Min(v, max);
Sse2.Store(p + i, v);
}
}
for (int i = 0; i < CacheLineCount; i += 8)
{
Vector128<short> p = Sse2.LoadVector128(pPtr + i);
Vector128<short> m = Sse2.LoadVector128(mPtr + i);
Vector128<short> great = Sse2.CompareGreaterThan(p, m);
Vector128<short> le = Sse42.Xor(great, Vector128.Create<short>(-1));
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ScanEvict(
byte* vPtr,
Vector128<byte> three
)
{
Vector128<byte> v = Sse2.LoadVector128(vPtr);
Vector128<byte> mask = Sse2.CompareEqual(v, three);
return Sse2.MoveMask(mask);
int mask = Sse42.MoveMask(le.AsByte());
while ((mask != 0) && (remain > 0))
{
int bit = BitOperations.TrailingZeroCount(mask);
int lane = bit >> 1;
if (owner.TryEvictAt(i + lane, pSpan)) {
remain--;
}
mask &= mask - 1;
mask &= mask - 1;
}
}
}
finally
{
predictors.ReleaseSpanOperation();
}
return remain;
}
}
StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+26 -12
View File
@@ -26,6 +26,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System.ComponentModel;
using System.Numerics;
namespace StgSharp.HighPerformance.Memory
{
@@ -55,7 +56,6 @@ namespace StgSharp.HighPerformance.Memory
return;
}
int missIndex = 0;
AgeAll();
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
@@ -82,26 +82,40 @@ namespace StgSharp.HighPerformance.Memory
{
head = (CacheLineHead*)handle;
index = head->Id;
(*_eviction)[index] = 0; // reset eviction counter
(*_predictCount)[index] = 0; // reset eviction counter
#region update predictor belonging
int predictId = head->Predictor;
IL4Predict belong = predictorSpan[predictId];
// TODO 由于L4的缓存行改为“可变长度”缓存行，需要调整缓存行复用策略
// This branch performs resident-line adoption rather than a simple policy
// equality check. Returning true means the current predictor can keep using
// the existing cache bytes without a write-back plus re-prefetch cycle.
if (!ReferenceEquals(predict, belong))
{
throw new L4PredictConflictException(head->Origin, head->Profile, predictId);
} else
{
// Compatibility failed, so the old owner must first flush its view of
// the line before the new predictor rematerializes the cache bytes.
_ = Interlocked.Increment(ref head->RemainingPredictionCount);
if ((*_predictCount)[index] > short.MaxValue / 2)
{
/*
* EXPERIMENTAL / INTENTIONALLY UNSAFE:
* This code performs a 32-bit atomic operation starting from a short reference.
*
* It is only valid in this specific storage layout because:
* - the runtime is little-endian;
* - arithmetic here does not cause harmful cross-16-bit carry/borrow;
* - the backing storage remains valid and accessible beyond the logical short-array boundary;
* - touching the adjacent 16 bits is acceptable for this allocation layout;
* - misaligned int access is acceptable on the target platform.
*
* This is safe only with the current L4 allocation/layout guarantees.
*/
short predictCount = Volatile.Read(ref (*_predictCount)[index]);
short mapCount = Volatile.Read(ref (*_mapCount)[index]);
int decrement = (short.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_predictCount)[index]), -decrement);
_ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_mapCount)[index]), -decrement);
}
(*_predictCount)[index]++;
}
#endregion
@@ -122,8 +136,8 @@ namespace StgSharp.HighPerformance.Memory
Thread.Sleep(1);
}
}
head = &_head[index];
head->_id = index;
head = _head + index;
head->Id = index;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->RefCount = 0;
StgSharp.Common/HighPerformance/Memory/L4.cs
+11 -8
View File
@@ -44,8 +44,9 @@ namespace StgSharp.HighPerformance.Memory
private const int PredictionLeastAhead = 1024;
private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
private readonly EvictionArray* _eviction;
private readonly CacheLineHead* _head;
private readonly MapArray* _mapCount;
private readonly PredictionArray* _predictCount;
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
@@ -75,13 +76,13 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
_baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
nuint ptr = _baseAddress;
AllocSize = entrySize;
@@ -98,8 +99,10 @@ namespace StgSharp.HighPerformance.Memory
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapSize;
_eviction = (EvictionArray*)ptr;
ptr += rrpvSize;
_predictCount = (PredictionArray*)ptr;
ptr += predictionCountSize;
_mapCount = (MapArray*)ptr;
ptr += predictionCountSize;
}
public nuint ByteCapacity { get; init; }
@@ -142,9 +145,9 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>(); // 16 byte align
nuint normalizedBufferSize = ByteCapacity; // align to 4096 byte, page size
Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
GC.SuppressFinalize(this);
}
@@ -179,7 +182,7 @@ namespace StgSharp.HighPerformance.Memory
CacheLineHead* entry = (CacheLineHead*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
(*_eviction)[entry->Id] = 0;
(*_predictCount)[entry->Id] = 0;
_ = Interlocked.Increment(ref _activeCount);
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref entry->RemainingPredictionCount);