diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
index af3edcf6..b871b479 100644
--- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
+++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs
@@ -35,10 +35,18 @@ namespace StgSharp.HighPerformance.Memory
{
[InlineArray(CacheLineCount)]
- private struct EvictionArray
+ private struct PredictionArray
{
- private byte pprv;
+ private short PredictionCount;
+
+ }
+
+ [InlineArray(CacheLineCount)]
+ private struct MapArray
+ {
+
+ private short MapCount;
}
@@ -51,8 +59,8 @@ namespace StgSharp.HighPerformance.Memory
#region reference counting and state information
[FieldOffset(52)] internal int RefCount;
- [FieldOffset(56)] internal int RemainingPredictionCount;
- [FieldOffset(60)] internal int Id;
+ [FieldOffset(60)] private readonly int Reverse;
+ [FieldOffset(56)] internal int Id;
#endregion
diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
index 039f7e17..4322dbc0 100644
--- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
+++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs
@@ -34,7 +34,6 @@ using System.Runtime.Intrinsics.X86;
using System.Text;
using System.Threading.Tasks;
-using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
using HlsfEntry = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator.Entry;
namespace StgSharp.HighPerformance.Memory
@@ -42,12 +41,6 @@ namespace StgSharp.HighPerformance.Memory
public partial class L4
{
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private unsafe void AgeAll()
- {
- _intrinsic.AgeAll(_eviction);
- }
-
///
///
///
@@ -62,41 +55,8 @@ namespace StgSharp.HighPerformance.Memory
int count = 1
)
{
- int remain = count;
- Vector128 three = Vector128.Create((byte)0x03);
- byte* p = (byte*)_eviction;
- _predictors.RequestSpanOperation();
- Span pSpan = _predictors.AsSpan();
- try
- {
- for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16)
- {
- int evictMask = _intrinsic.ScanEvict(p + i, three);
- while ((evictMask != 0) && (remain > 0))
- {
- int bitPos = BitOperations.TrailingZeroCount(evictMask);
- int index = bitPos + i;
- CacheLineHead* head = &_head[index];
- if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
- {
- // EvictLine(i + index);
- _ = _map.TryRemove(head->Origin, out _);
- int id = head->Predictor;
- IL4Predict predict = pSpan[id] ??
- throw new L4PredictUnregisteredException(id);
- predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan((byte*)head->Address, (int)head->Size));
- RecycleLine(index);
- remain--;
- }
- evictMask &= evictMask - 1;
- }
- }
- return remain;
- }
- finally
- {
- _predictors.ReleaseSpanOperation();
- }
+ count = (count < 0) ? 0 : count;
+ return _intrinsic.ScanEvict((short*)_predictCount, (short*)_mapCount, count, this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -108,6 +68,29 @@ namespace StgSharp.HighPerformance.Memory
_bufferAllocator.Free(entry);
}
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private unsafe bool TryEvictAt(
+ int index,
+ Span predictorSpan
+ )
+ {
+ CacheLineHead* head = &_head[index];
+ if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
+ {
+ // EvictLine(i + index);
+ _ = _map.TryRemove(head->Origin, out _);
+ int id = head->Predictor;
+ IL4Predict predict = predictorSpan[id] ??
+ throw new L4PredictUnregisteredException(id);
+ predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan((byte*)head->Address, (int)head->Size));
+ RecycleLine(index);
+ (*_mapCount)[index] = 0;
+ (*_predictCount)[index] = short.MaxValue;
+ return true;
+ }
+ return false;
+ }
+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryGetEmptyLine(
nuint size,
@@ -120,8 +103,8 @@ namespace StgSharp.HighPerformance.Memory
index = 0;
return false;
}
- Hlsf.Entry* entry = handle.EntryHandle;
- index = (int)(entry - ((HlsfEntry*)_entryManager._basePtr));
+ CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
+ index = entry->Id;
return true;
}
diff --git a/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs b/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs
index 0bef7e23..5502302b 100644
--- a/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs
+++ b/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs
@@ -25,6 +25,7 @@
//
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
+using StgSharp.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -44,15 +45,12 @@ namespace StgSharp.HighPerformance.Memory
static IIntrinsicKernel Instance { get; } = Create();
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public void AgeAll(
- EvictionArray* eviction
- );
-
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int ScanEvict(
- byte* vPtr,
- Vector128 three
+ short* pPtr,
+ short* mPtr,
+ int count,
+ L4 owner
);
private static IIntrinsicKernel Create()
@@ -68,35 +66,49 @@ namespace StgSharp.HighPerformance.Memory
}
- private struct SseKernel : IIntrinsicKernel
+ private readonly struct SseKernel : IIntrinsicKernel
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public void AgeAll(
- EvictionArray* eviction
+ public readonly int ScanEvict(
+ short* pPtr,
+ short* mPtr,
+ int count,
+ L4 owner
)
{
- Vector128 one = Vector128.Create((byte)0x01);
- Vector128 max = Vector128.Create((byte)0x03);
- byte* p = (byte*)eviction;
- for (int i = 0; i < CacheLineCount; i += 16)
+ int remain = count;
+ ConcurrentFlexibleArray predictors = owner._predictors;
+ predictors.RequestSpanOperation();
+ Span pSpan = predictors.AsSpan();
+ try
{
- Vector128 v = Sse2.LoadVector128(p + i);
- v = Sse2.AddSaturate(v, one);
- v = Sse2.Min(v, max);
- Sse2.Store(p + i, v);
- }
- }
+ for (int i = 0; i < CacheLineCount; i += 8)
+ {
+ Vector128 p = Sse2.LoadVector128(pPtr + i);
+ Vector128 m = Sse2.LoadVector128(mPtr + i);
+ Vector128 great = Sse2.CompareGreaterThan(p, m);
+ Vector128 le = Sse42.Xor(great, Vector128.Create(-1));
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public int ScanEvict(
- byte* vPtr,
- Vector128 three
- )
- {
- Vector128 v = Sse2.LoadVector128(vPtr);
- Vector128 mask = Sse2.CompareEqual(v, three);
- return Sse2.MoveMask(mask);
+ int mask = Sse42.MoveMask(le.AsByte());
+ while ((mask != 0) && (remain > 0))
+ {
+ int bit = BitOperations.TrailingZeroCount(mask);
+ int lane = bit >> 1;
+
+ if (owner.TryEvictAt(i + lane, pSpan)) {
+ remain--;
+ }
+ mask &= mask - 1;
+ mask &= mask - 1;
+ }
+ }
+ }
+ finally
+ {
+ predictors.ReleaseSpanOperation();
+ }
+ return remain;
}
}
diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
index def8d25a..56835b88 100644
--- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
+++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs
@@ -26,6 +26,7 @@
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using System.ComponentModel;
+using System.Numerics;
namespace StgSharp.HighPerformance.Memory
{
@@ -55,7 +56,6 @@ namespace StgSharp.HighPerformance.Memory
return;
}
int missIndex = 0;
- AgeAll();
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
@@ -82,26 +82,40 @@ namespace StgSharp.HighPerformance.Memory
{
head = (CacheLineHead*)handle;
index = head->Id;
- (*_eviction)[index] = 0; // reset eviction counter
+ (*_predictCount)[index] = 0; // reset eviction counter
#region update predictor belonging
int predictId = head->Predictor;
IL4Predict belong = predictorSpan[predictId];
- // TODO 由于L4的缓存行改为“可变长度”缓存行，需要调整缓存行复用策略
- // This branch performs resident-line adoption rather than a simple policy
- // equality check. Returning true means the current predictor can keep using
- // the existing cache bytes without a write-back plus re-prefetch cycle.
if (!ReferenceEquals(predict, belong))
{
throw new L4PredictConflictException(head->Origin, head->Profile, predictId);
} else
{
- // Compatibility failed, so the old owner must first flush its view of
- // the line before the new predictor rematerializes the cache bytes.
-
- _ = Interlocked.Increment(ref head->RemainingPredictionCount);
+ if ((*_predictCount)[index] > short.MaxValue / 2)
+ {
+ /*
+ * EXPERIMENTAL / INTENTIONALLY UNSAFE:
+ * This code performs a 32-bit atomic operation starting from a short reference.
+ *
+ * It is only valid in this specific storage layout because:
+ * - the runtime is little-endian;
+ * - arithmetic here does not cause harmful cross-16-bit carry/borrow;
+ * - the backing storage remains valid and accessible beyond the logical short-array boundary;
+ * - touching the adjacent 16 bits is acceptable for this allocation layout;
+ * - misaligned int access is acceptable on the target platform.
+ *
+ * This is safe only with the current L4 allocation/layout guarantees.
+ */
+ short predictCount = Volatile.Read(ref (*_predictCount)[index]);
+ short mapCount = Volatile.Read(ref (*_mapCount)[index]);
+ int decrement = (short.Min(predictCount, mapCount) + 1) / 2;
+ _ = Interlocked.Add(ref Unsafe.As(ref (*_predictCount)[index]), -decrement);
+ _ = Interlocked.Add(ref Unsafe.As(ref (*_mapCount)[index]), -decrement);
+ }
+ (*_predictCount)[index]++;
}
#endregion
@@ -122,8 +136,8 @@ namespace StgSharp.HighPerformance.Memory
Thread.Sleep(1);
}
}
- head = &_head[index];
- head->_id = index;
+ head = _head + index;
+ head->Id = index;
head->Origin = address;
head->Profile = pHandle.MapPolicy;
head->RefCount = 0;
diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs
index fe7ac5f7..75f9df92 100644
--- a/StgSharp.Common/HighPerformance/Memory/L4.cs
+++ b/StgSharp.Common/HighPerformance/Memory/L4.cs
@@ -44,8 +44,9 @@ namespace StgSharp.HighPerformance.Memory
private const int PredictionLeastAhead = 1024;
private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
- private readonly EvictionArray* _eviction;
private readonly CacheLineHead* _head;
+ private readonly MapArray* _mapCount;
+ private readonly PredictionArray* _predictCount;
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
@@ -75,13 +76,13 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
- nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align
+ nuint predictionCountSize = (nuint)Unsafe.SizeOf(); // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
- _baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
+ _baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
nuint ptr = _baseAddress;
AllocSize = entrySize;
@@ -98,8 +99,10 @@ namespace StgSharp.HighPerformance.Memory
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapSize;
- _eviction = (EvictionArray*)ptr;
- ptr += rrpvSize;
+ _predictCount = (PredictionArray*)ptr;
+ ptr += predictionCountSize;
+ _mapCount = (MapArray*)ptr;
+ ptr += predictionCountSize;
}
public nuint ByteCapacity { get; init; }
@@ -142,9 +145,9 @@ namespace StgSharp.HighPerformance.Memory
nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align
nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
- nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align
+ nuint predictionCountSize = (nuint)Unsafe.SizeOf(); // 16 byte align
nuint normalizedBufferSize = ByteCapacity; // align to 4096 byte, page size
- Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
+ Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
GC.SuppressFinalize(this);
}
@@ -179,7 +182,7 @@ namespace StgSharp.HighPerformance.Memory
CacheLineHead* entry = (CacheLineHead*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
- (*_eviction)[entry->Id] = 0;
+ (*_predictCount)[entry->Id] = 0;
_ = Interlocked.Increment(ref _activeCount);
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref entry->RemainingPredictionCount);