some chore

StgSharp.Common/HighPerformance/Memory/L4.Allocate.cs

@@ -1,53 +0,0 @@
-//-----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-// file="L4.Allocate.cs"
-// Project: StgSharp
-// AuthorGroup: Nitload Space
-// Copyright (c) Nitload Space. All rights reserved.
-//     
-// Permission is hereby granted, free of charge, to any person 
-// obtaining a copy of this software and associated documentation 
-// files (the “Software”), to deal in the Software without restriction, 
-// including without limitation the rights to use, copy, modify, merge,
-// publish, distribute, sublicense, and/or sell copies of the Software, 
-// and to permit persons to whom the Software is furnished to do so, 
-// subject to the following conditions:
-//     
-// The above copyright notice and 
-// this permission notice shall be included in all copies 
-// or substantial portions of the Software.
-//     
-// THE SOFTWARE IS PROVIDED “AS IS”, 
-// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
-// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
-// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 
-// ARISING FROM, OUT OF OR IN CONNECTION WITH 
-// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-//     
-// -----------------------------------------------------------------------
-// -----------------------------------------------------------------------
-namespace StgSharp.HighPerformance.Memory
-{
-    public partial class L4
-    {
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private void RecycleLine(int id)
-        {
-            _idMux.Push(id);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private bool TryGetEmptyLine(out int index)
-        {
-            if (_idMux.TryPop(out index)) {
-                return true;
-            }
-            index = _maxId;
-            return Interlocked.Increment(ref _maxId) >= CacheLineCount;
-        }
-
-    }
-}

StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs

@@ -40,21 +40,6 @@ namespace StgSharp.HighPerformance.Memory
     public partial class L4
     {
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private unsafe void AgeAll()
-        {
-            Vector128<byte> one = Vector128.Create((byte)0x01);
-            Vector128<byte> max = Vector128.Create((byte)0x03);
-            byte* p = (byte*)_eviction;
-            for (int i = 0; i < CacheLineCount; i += 16)
-            {
-                Vector128<byte> v = Sse2.LoadVector128(p + i);
-                v = Sse2.AddSaturate(v, one);
-                v = Sse2.Min(v, max);
-                Sse2.Store(p + i, v);
-            }
-        }
-
         [InlineArray(CacheLineCount)]
         private struct EvictionArray
         {

StgSharp.Common/HighPerformance/Memory/L4.Evict.cs

@@ -28,6 +28,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Numerics;
 using System.Runtime.Intrinsics;
 using System.Runtime.Intrinsics.X86;
 using System.Text;
@@ -69,13 +70,40 @@ namespace StgSharp.HighPerformance.Memory
                 int evictMask = Sse2.MoveMask(mask);
                 while (evictMask != 0 && remain > 0)
                 {
-                    int index = BitOperations.TrailingZeroCount(evictMask);
+                    int bitPos = BitOperations.TrailingZeroCount(evictMask);
-                    evictMask &= (evictMask - 1);
+                    int index = bitPos + i;
-                    EvictLine(i + index);
+                    CacheLineHead* head = &_head[index];
-                    remain--;
+                    if (Interlocked.CompareExchange(ref head->_refCount, -1, 0) == 0)
+                    {
+                        // EvictLine(i + index);
+                        _ = _map.TryRemove(head->_origin, out _);
+                        RecycleLine(index);
+                        remain--;
+                    }
+                    evictMask &= evictMask - 1;
                 }
             }
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void RecycleLine(
+                     int id
+        )
+        {
+            _idMux.Push(id);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool TryGetEmptyLine(
+                     out int index
+        )
+        {
+            if (_idMux.TryPop(out index)) {
+                return true;
+            }
+            index = _maxId;
+            return Interlocked.Increment(ref _maxId) >= CacheLineCount;
+        }
+
     }
 }

StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="L4.Prefetch"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -63,15 +63,19 @@ namespace StgSharp.HighPerformance.Memory
             {
                 ref CacheLinePrediction handle = ref span[i];
                 nuint address = handle.Address;
-                nuint mapPtr = _cacheLineAllocator.Allocate();
+                if (!TryGetEmptyLine(out int index)) {
-                CacheLineHead* head = (CacheLineHead*)_entryAllocator.Allocate();
+                    throw new OverflowException("L4 has no empty cache line.");
+                }
+                CacheLineHead* head = &_head[index];
 
 
                 // pre fetch them here
             }
         }
 
-        private partial bool TestHit(nuint origin)
+        private partial bool TestHit(
+                             nuint origin
+        )
         {
             if (_map.TryGet(origin, out nuint handle))
             {

StgSharp.Common/HighPerformance/Memory/L4.cs

@@ -2,8 +2,8 @@
 // -----------------------------------------------------------------------
 // file="L4"
 // Project: StgSharp
-// AuthorGroup: Nitload Space
+// AuthorGroup: Nitload
-// Copyright (c) Nitload Space. All rights reserved.
+// Copyright (c) Nitload. All rights reserved.
 //     
 // Permission is hereby granted, free of charge, to any person obtaining a copy
 // of this software and associated documentation files (the "Software"), to deal
@@ -26,18 +26,12 @@
 // -----------------------------------------------------------------------
 // -----------------------------------------------------------------------
 using StgSharp.Collections;
-using StgSharp.Mathematics.Memory;
+using System.Data;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Runtime.InteropServices;
-using System.Text;
-using System.Threading.Tasks;
-using System.Xml.Serialization;
 
 namespace StgSharp.HighPerformance.Memory
 {
-    public unsafe partial class L4 : IDisposable
+    public sealed unsafe partial class L4 : IDisposable
     {
 
         private const int CacheLineCount = 8192;
@@ -55,25 +49,40 @@ namespace StgSharp.HighPerformance.Memory
 
         private int _aheadCount;
         private int _maxId;
+        private volatile int _prefetching;
         private readonly nuint _baseAddress;
         private readonly SwissTable _map;
         private readonly UnmanagedStack<int> _idMux;
 
         public L4()
         {
-            int cacheSize = Unsafe.SizeOf<CacheLineData>() * CacheLineCount;
+            // init order is fallow by align requirement of each component.
-            int entrySize = Unsafe.SizeOf<CacheLineHead>() * CacheLineCount;
+            nuint cacheSize = (nuint)Unsafe.SizeOf<CacheLineData>() * CacheLineCount;   // SIMDID, no less than 16 byte bit
-            int stackSize = sizeof(int) * CacheLineCount;
+            nuint entrySize = (nuint)Unsafe.SizeOf<CacheLineHead>() * CacheLineCount;   // 16 byte align
-            int mapSize = SwissTable.RequestedNativeMemorySize;
+            nuint predictionSize = (nuint)Unsafe.SizeOf<CacheLinePrediction>() * PredictionCount;   //16 byte align
-            int predictionSize = Unsafe.SizeOf<CacheLinePrediction>() * PredictionCount;
+            nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize;     // 16 byte align
-            int rrpvSize = Unsafe.SizeOf<EvictionArray>();
+            nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>();           // 16 byte align
-            _baseAddress = (nuint)NativeMemory.AlignedAlloc((nuint)(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize), 16);
+            nuint stackSize = (nuint)sizeof(int) * CacheLineCount;                      // int align
+
+            // malloc all memory at once, and free at once in Dispose.
+            // This is to avoid fragmentation and improve performance.
+            _baseAddress = (nuint)NativeMemory.AlignedAlloc(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize, 16);
+            nuint ptr = _baseAddress;
+
+            // init all components with the buffer, and move the buffer pointer forward.
             nuint _buffer = _baseAddress;
             _data = (CacheLineData*)_buffer;
-            _head = (CacheLineHead*)(_buffer + (nuint)cacheSize);
+            ptr += cacheSize;
-            _prediction = (CacheLinePrediction*)(_buffer + (nuint)(cacheSize + entrySize));
+            _head = (CacheLineHead*)ptr;
-            _map = SwissTable.Create(_baseAddress + (nuint)(cacheSize + entrySize), _ => { });
+            ptr += entrySize;
-            _eviction = (EvictionArray*)(_baseAddress + (nuint)(cacheSize + entrySize + mapSize));
+            _prediction = (CacheLinePrediction*)ptr;
+            ptr += predictionSize;
+            _map = SwissTable.Create(_baseAddress + cacheSize + entrySize, _ => { });
+            _prediction = (CacheLinePrediction*)ptr;
+            _eviction = (EvictionArray*)ptr;
+            ptr += rrpvSize;
+            _idMux = new UnmanagedStack<int>((int*)ptr, CacheLineCount, _ => { });
+            ptr += stackSize;
         }
 
         public IPrefetchPredict PrefetchPredict { get; set; }
@@ -93,30 +102,58 @@ namespace StgSharp.HighPerformance.Memory
             }
             _disposed = true;
             _map.Dispose();
+            _idMux.Dispose();
             NativeMemory.AlignedFree((void*)_baseAddress);
             GC.SuppressFinalize(this);
         }
 
-        public CacheLine Map(nuint origin)
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public CacheLine Map(
+                         nuint origin
+        )
         {
-            if (_map.TryGet(origin, out nuint entryHandle))
+            if (_aheadCount < PredictionLeastAhead)
             {
-                // cache hit
+                while (true)
-                CacheLineHead* entry = (CacheLineHead*)entryHandle;
+                {
-                return new CacheLine(entry->_address, ref _activeCount, ref entry->_refCount);
+                    if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
+                    {
+                        if (_aheadCount < PredictionLeastAhead)
+                        {
+                            // do prefetch in work load now, sync
+                            Prefetch();
+                            break;
+                        } else
+                        {
+                            _prefetching = 0;
+                        }
+                    }
+                }
+            }
+            while (true)
+            {
+                if (_map.TryGet(origin, out nuint entryHandle))
+                {
+                    // cache hit
+                    CacheLineHead* entry = (CacheLineHead*)entryHandle;
+                    if (Interlocked.Increment(ref entry->_refCount) < 0)
+                    {
+                        (*_eviction)[entry->_id] = 0;
+                        return new CacheLine(entry->_address, ref _activeCount, ref entry->_refCount);
+                    } else
+                    {
+                        _ = Interlocked.Decrement(ref entry->_refCount);
+                    }
+                }
+                Thread.Sleep(0);
             }
-
-            // cache miss
-            /*
-             * post command
-             * then wait for completion
-             */
-            return /*origin/**/default;   //error now
         }
 
         private partial void Prefetch();
 
-        private partial bool TestHit(nuint origin);
+        private partial bool TestHit(
+                             nuint origin
+        );
 
         public ref struct CacheLine : IDisposable
         {
@@ -125,7 +162,11 @@ namespace StgSharp.HighPerformance.Memory
             private ref int _refCount;
             private nuint _handle;
 
-            internal CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount)
+            internal CacheLine(
+                     nuint handle,
+                     ref int activeCount,
+                     ref int cacheRefCount
+            )
             {
                 _handle = handle;
                 _refCount = ref cacheRefCount;
@@ -134,6 +175,9 @@ namespace StgSharp.HighPerformance.Memory
                 _ = Interlocked.Increment(ref _refCount);
             }
 
+            [Obsolete("You should not create a CacheLineHandle outside a L4", true)]
+            public CacheLine() { }
+
             public void Dispose()
             {
                 if (_handle == 0) {