L4 is now on new algo, using new eviction arrays, using union of hlsf and cache line head.

StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs

@@ -35,10 +35,18 @@ namespace StgSharp.HighPerformance.Memory
     {
 
         [InlineArray(CacheLineCount)]
-        private struct EvictionArray
+        private struct PredictionArray
         {
 
-            private byte pprv;
+            private short PredictionCount;
+
+        }
+
+        [InlineArray(CacheLineCount)]
+        private struct MapArray
+        {
+
+            private short MapCount;
 
         }
 
@@ -51,8 +59,8 @@ namespace StgSharp.HighPerformance.Memory
             #region reference counting and state information
 
             [FieldOffset(52)] internal int RefCount;
-            [FieldOffset(56)] internal int RemainingPredictionCount;
+            [FieldOffset(60)] private readonly int Reverse;
-            [FieldOffset(60)] internal int Id;
+            [FieldOffset(56)] internal int Id;
 
             #endregion
 

StgSharp.Common/HighPerformance/Memory/L4.Evict.cs

@@ -34,7 +34,6 @@ using System.Runtime.Intrinsics.X86;
 using System.Text;
 using System.Threading.Tasks;
 
-using Hlsf = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator;
 using HlsfEntry = StgSharp.HighPerformance.Memory.HybridLayerSegregatedFitAllocator.Entry;
 
 namespace StgSharp.HighPerformance.Memory
@@ -42,12 +41,6 @@ namespace StgSharp.HighPerformance.Memory
     public partial class L4
     {
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private unsafe void AgeAll()
-        {
-            _intrinsic.AgeAll(_eviction);
-        }
-
         /// <summary>
         ///
         /// </summary>
@@ -62,41 +55,8 @@ namespace StgSharp.HighPerformance.Memory
                            int count = 1
         )
         {
-            int remain = count;
+            count = (count < 0) ? 0 : count;
-            Vector128<byte> three = Vector128.Create((byte)0x03);
+            return _intrinsic.ScanEvict((short*)_predictCount, (short*)_mapCount, count, this);
-            byte* p = (byte*)_eviction;
-            _predictors.RequestSpanOperation();
-            Span<IL4Predict> pSpan = _predictors.AsSpan();
-            try
-            {
-                for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16)
-                {
-                    int evictMask = _intrinsic.ScanEvict(p + i, three);
-                    while ((evictMask != 0) && (remain > 0))
-                    {
-                        int bitPos = BitOperations.TrailingZeroCount(evictMask);
-                        int index = bitPos + i;
-                        CacheLineHead* head = &_head[index];
-                        if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
-                        {
-                            // EvictLine(i + index);
-                            _ = _map.TryRemove(head->Origin, out _);
-                            int id = head->Predictor;
-                            IL4Predict predict = pSpan[id] ??
-                                throw new L4PredictUnregisteredException(id);
-                            predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
-                            RecycleLine(index);
-                            remain--;
-                        }
-                        evictMask &= evictMask - 1;
-                    }
-                }
-                return remain;
-            }
-            finally
-            {
-                _predictors.ReleaseSpanOperation();
-            }
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -108,6 +68,29 @@ namespace StgSharp.HighPerformance.Memory
             _bufferAllocator.Free(entry);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private unsafe bool TryEvictAt(
+                            int index,
+                            Span<IL4Predict> predictorSpan
+        )
+        {
+            CacheLineHead* head = &_head[index];
+            if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
+            {
+                // EvictLine(i + index);
+                _ = _map.TryRemove(head->Origin, out _);
+                int id = head->Predictor;
+                IL4Predict predict = predictorSpan[id] ??
+                    throw new L4PredictUnregisteredException(id);
+                predict.WriteBack(head->Origin, head->Profile, new ReadOnlySpan<byte>((byte*)head->Address, (int)head->Size));
+                RecycleLine(index);
+                (*_mapCount)[index] = 0;
+                (*_predictCount)[index] = short.MaxValue;
+                return true;
+            }
+            return false;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private unsafe bool TryGetEmptyLine(
                             nuint size,
@@ -120,8 +103,8 @@ namespace StgSharp.HighPerformance.Memory
                 index = 0;
                 return false;
             }
-            Hlsf.Entry* entry = handle.EntryHandle;
+            CacheLineHead* entry = (CacheLineHead*)handle.EntryHandle;
-            index = (int)(entry - ((HlsfEntry*)_entryManager._basePtr));
+            index = entry->Id;
             return true;
         }
 

StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs

@@ -25,6 +25,7 @@
 //     
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
+using StgSharp.Collections;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -44,15 +45,12 @@ namespace StgSharp.HighPerformance.Memory
 
             static IIntrinsicKernel Instance { get; } = Create();
 
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void AgeAll(
-                        EvictionArray* eviction
-            );
-
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
             public int ScanEvict(
-                       byte* vPtr,
+                       short* pPtr,
-                       Vector128<byte> three
+                       short* mPtr,
+                       int count,
+                       L4 owner
             );
 
             private static IIntrinsicKernel Create()
@@ -68,35 +66,49 @@ namespace StgSharp.HighPerformance.Memory
 
         }
 
-        private struct SseKernel : IIntrinsicKernel
+        private readonly struct SseKernel : IIntrinsicKernel
         {
 
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            public void AgeAll(
+            public readonly int ScanEvict(
-                        EvictionArray* eviction
+                                short* pPtr,
+                                short* mPtr,
+                                int count,
+                                L4 owner
             )
             {
-                Vector128<byte> one = Vector128.Create((byte)0x01);
+                int remain = count;
-                Vector128<byte> max = Vector128.Create((byte)0x03);
+                ConcurrentFlexibleArray<IL4Predict> predictors = owner._predictors;
-                byte* p = (byte*)eviction;
+                predictors.RequestSpanOperation();
-                for (int i = 0; i < CacheLineCount; i += 16)
+                Span<IL4Predict> pSpan = predictors.AsSpan();
+                try
                 {
-                    Vector128<byte> v = Sse2.LoadVector128(p + i);
+                    for (int i = 0; i < CacheLineCount; i += 8)
-                    v = Sse2.AddSaturate(v, one);
+                    {
-                    v = Sse2.Min(v, max);
+                        Vector128<short> p = Sse2.LoadVector128(pPtr + i);
-                    Sse2.Store(p + i, v);
+                        Vector128<short> m = Sse2.LoadVector128(mPtr + i);
-                }
+                        Vector128<short> great = Sse2.CompareGreaterThan(p, m);
-            }
+                        Vector128<short> le = Sse42.Xor(great, Vector128.Create<short>(-1));
 
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+                        int mask = Sse42.MoveMask(le.AsByte());
-            public int ScanEvict(
+                        while ((mask != 0) && (remain > 0))
-                       byte* vPtr,
+                        {
-                       Vector128<byte> three
+                            int bit = BitOperations.TrailingZeroCount(mask);
-            )
+                            int lane = bit >> 1;
-            {
+
-                Vector128<byte> v = Sse2.LoadVector128(vPtr);
+                            if (owner.TryEvictAt(i + lane, pSpan)) {
-                Vector128<byte> mask = Sse2.CompareEqual(v, three);
+                                remain--;
-                return Sse2.MoveMask(mask);
+                            }
+                            mask &= mask - 1;
+                            mask &= mask - 1;
+                        }
+                    }
+                }
+                finally
+                {
+                    predictors.ReleaseSpanOperation();
+                }
+                return remain;
             }
 
         }

StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs

@@ -26,6 +26,7 @@
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 using System.ComponentModel;
+using System.Numerics;
 
 namespace StgSharp.HighPerformance.Memory
 {
@@ -55,7 +56,6 @@ namespace StgSharp.HighPerformance.Memory
                 return;
             }
             int missIndex = 0;
-            AgeAll();
             for (int i = 0; i < actualCount; i++)
             {
                 ref CacheLineDescription handle = ref span[i];
@@ -82,26 +82,40 @@ namespace StgSharp.HighPerformance.Memory
                     {
                         head = (CacheLineHead*)handle;
                         index = head->Id;
-                        (*_eviction)[index] = 0; // reset eviction counter
+                        (*_predictCount)[index] = 0; // reset eviction counter
 
                         #region update predictor belonging
 
                         int predictId = head->Predictor;
                         IL4Predict belong = predictorSpan[predictId];
 
-                        // TODO 由于L4的缓存行改为“可变长度”缓存行，需要调整缓存行复用策略
-                        // This branch performs resident-line adoption rather than a simple policy
-                        // equality check. Returning true means the current predictor can keep using
-                        // the existing cache bytes without a write-back plus re-prefetch cycle.
                         if (!ReferenceEquals(predict, belong))
                         {
                             throw new L4PredictConflictException(head->Origin, head->Profile, predictId);
                         } else
                         {
-                            // Compatibility failed, so the old owner must first flush its view of
+                            if ((*_predictCount)[index] > short.MaxValue / 2)
-                            // the line before the new predictor rematerializes the cache bytes.
+                            {
-
+                                /*
-                            _ = Interlocked.Increment(ref head->RemainingPredictionCount);
+                                 * EXPERIMENTAL / INTENTIONALLY UNSAFE:
+                                 * This code performs a 32-bit atomic operation starting from a short reference.
+                                 *
+                                 * It is only valid in this specific storage layout because:
+                                 * - the runtime is little-endian;
+                                 * - arithmetic here does not cause harmful cross-16-bit carry/borrow;
+                                 * - the backing storage remains valid and accessible beyond the logical short-array boundary;
+                                 * - touching the adjacent 16 bits is acceptable for this allocation layout;
+                                 * - misaligned int access is acceptable on the target platform.
+                                 *
+                                 * This is safe only with the current L4 allocation/layout guarantees.
+                                 */
+                                short predictCount = Volatile.Read(ref (*_predictCount)[index]);
+                                short mapCount = Volatile.Read(ref (*_mapCount)[index]);
+                                int decrement = (short.Min(predictCount, mapCount) + 1) / 2;
+                                _ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_predictCount)[index]), -decrement);
+                                _ = Interlocked.Add(ref Unsafe.As<short, int>(ref (*_mapCount)[index]), -decrement);
+                            }
+                            (*_predictCount)[index]++;
                         }
 
                         #endregion
@@ -122,8 +136,8 @@ namespace StgSharp.HighPerformance.Memory
                             Thread.Sleep(1);
                         }
                     }
-                    head = &_head[index];
+                    head = _head + index;
-                    head->_id = index;
+                    head->Id = index;
                     head->Origin = address;
                     head->Profile = pHandle.MapPolicy;
                     head->RefCount = 0;

StgSharp.Common/HighPerformance/Memory/L4.cs

@@ -44,8 +44,9 @@ namespace StgSharp.HighPerformance.Memory
         private const int PredictionLeastAhead = 1024;
 
         private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
-        private readonly EvictionArray* _eviction;
         private readonly CacheLineHead* _head;
+        private readonly MapArray* _mapCount;
+        private readonly PredictionArray* _predictCount;
         private readonly CacheLineDescription* _prediction;
         private bool _disposed;
         private bool _exhausted;
@@ -75,13 +76,13 @@ namespace StgSharp.HighPerformance.Memory
             nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount;   // 16 byte align
             nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount;   //16 byte align
             nuint mapSize = SwissTable.RequestedNativeMemorySize;     // 16 byte align
-            nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>();           // 16 byte align
+            nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>();           // 16 byte align
             nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));    // align to 4096 byte, page size
             ByteCapacity = normalizedBufferSize;
 
             // malloc all memory at once, and free at once in Dispose.
             // This is to avoid fragmentation and improve performance.
-            _baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
+            _baseAddress = (nuint)Numa.Alloc(bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
             nuint ptr = _baseAddress;
 
             AllocSize = entrySize;
@@ -98,8 +99,10 @@ namespace StgSharp.HighPerformance.Memory
             ptr += predictionSize;
             _map = SwissTable.Create(ptr, _ => { });
             ptr += mapSize;
-            _eviction = (EvictionArray*)ptr;
+            _predictCount = (PredictionArray*)ptr;
-            ptr += rrpvSize;
+            ptr += predictionCountSize;
+            _mapCount = (MapArray*)ptr;
+            ptr += predictionCountSize;
         }
 
         public nuint ByteCapacity { get; init; }
@@ -142,9 +145,9 @@ namespace StgSharp.HighPerformance.Memory
             nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount;   // 16 byte align
             nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount;   //16 byte align
             nuint mapSize = SwissTable.RequestedNativeMemorySize;     // 16 byte align
-            nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>();           // 16 byte align
+            nuint predictionCountSize = (nuint)Unsafe.SizeOf<PredictionArray>();           // 16 byte align
             nuint normalizedBufferSize = ByteCapacity;    // align to 4096 byte, page size
-            Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + rrpvSize + normalizedBufferSize);
+            Numa.Free((void*)_baseAddress, _bindedNuma, entrySize + headSize + mapSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize);
             GC.SuppressFinalize(this);
         }
 
@@ -179,7 +182,7 @@ namespace StgSharp.HighPerformance.Memory
                     CacheLineHead* entry = (CacheLineHead*)entryHandle;
                     if (Interlocked.Increment(ref entry->RefCount) > 0)
                     {
-                        (*_eviction)[entry->Id] = 0;
+                        (*_predictCount)[entry->Id] = 0;
                         _ = Interlocked.Increment(ref _activeCount);
                         _ = Interlocked.Decrement(ref _aheadCount);
                         _ = Interlocked.Decrement(ref entry->RemainingPredictionCount);