switch root namespace name to NITLOAD

src/Nitload.Common/HighPerformance/Memory/L4.cs

@@ -0,0 +1,188 @@
+// -----------------------------------------------------------------------------
+// file="L4"
+// Project: StgSharp
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using Nitload.Common.Collections;
+using Nitload.Common.HighPerformance.Memory;
+using Nitload.Common.HighPerformance.ProcessorAbstraction;
+using System.Data;
+using System.Runtime.InteropServices;
+
+namespace Nitload.Common.HighPerformance.Memory
+{
+    public sealed unsafe partial class L4 : IDisposable
+    {
+
+        private const int CacheLineCount = 8192;
+        private const nuint PageSize = 4096;
+        private const int PredictionCount = 2048;
+        private const int PredictionLeastAhead = 1024;
+
+        /// <summary>
+        ///   Array of metadata for each cache line, also used as metadata for tlsf allocator.
+        /// </summary>
+        private readonly CacheLineMetadata* _head;
+        /// <summary>
+        ///   Count of cache lines mapped for each prediction, used for eviction policy.
+        /// </summary>
+        private readonly int* _mapCount;
+        /// <summary>
+        ///   Count of times each cache line is predicted, used for eviction policy.
+        /// </summary>
+        private readonly int* _predictCount;
+        /// <summary>
+        ///   Original memory view for prediction span.
+        /// </summary>
+        private readonly CacheLineDescription* _prediction;
+        private bool _disposed;
+        private bool _exhausted;
+
+        private int _aheadCount;
+
+        private readonly int _bindedNuma;
+        private volatile int _prefetching;
+        private readonly nuint _baseAddress;
+        private readonly SwissTable _map;
+
+        private readonly TLSF _bufferAllocator;
+
+        public L4(
+               int bindedNuma,
+               uint bufferSize = 8192 * 8192,
+               int align = 16,
+               uint maxCacheLineSize = 8192
+        )
+        {
+            ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
+            ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
+            _bindedNuma = bindedNuma;
+
+            // init order is fallow by align requirement of each component.
+            nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount;   // 16 byte align
+            nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount;   //16 byte align
+            nuint mapCountSize = SwissTable.RequestedNativeMemorySize;     // 16 byte align
+            nuint predictionCountSize = sizeof(int) * CacheLineCount;           // 16 byte align
+            nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));    // align to 4096 byte, page size
+            ByteCapacity = normalizedBufferSize;
+
+            AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
+
+            // malloc all memory at once, and free at once in Dispose.
+            // This is to avoid fragmentation and improve performance.
+            _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
+            nuint ptr = _baseAddress;
+
+            // init all components with the buffer, and move the buffer pointer forward.
+            nuint _buffer = _baseAddress;
+            _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
+            _buffer += normalizedBufferSize;
+            _head = (CacheLineMetadata*)ptr;
+            ptr += headSize;
+            _prediction = (CacheLineDescription*)ptr;
+            ptr += predictionSize;
+            _map = SwissTable.Create(ptr, _ => { });
+            ptr += mapCountSize;
+            _predictCount = (int*)ptr;
+            ptr += predictionCountSize;
+            _mapCount = (int*)ptr;
+            ptr += predictionCountSize;
+
+            M512* mPtr = (M512*)_head;
+            for (uint i = 1; i < CacheLineCount; i++) {
+                mPtr->BroadCastFrom<ulong>(0);
+            }
+        }
+
+        public nuint ByteCapacity { get; init; }
+
+        public nuint AllocSize { get; init; }
+
+        // public IL4Predict PrefetchPredict { get; set; }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void ContinueOnce(
+                    Handle line
+        )
+        {
+            if (line.Address > 0) {
+                _ = Interlocked.Decrement(ref _aheadCount);
+            }
+        }
+
+        [System.Diagnostics.CodeAnalysis.SuppressMessage(
+                "Design",
+                "CA1063:Implement IDisposable correctly",
+                Justification = """
+                    Only owns native memory via Numa.Free();
+                    no managed IDisposable fields.
+                    Dispose(bool) pattern is unnecessary.
+                    """)]
+        public void Dispose()
+        {
+            if (_disposed) {
+                return;
+            }
+            _disposed = true;
+            _map.Dispose();
+#pragma warning disable CA1816
+            GC.SuppressFinalize(_bufferAllocator);
+#pragma warning restore CA1816
+
+
+            // init order is fallow by align requirement of each component.
+            Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
+            GC.SuppressFinalize(this);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Handle Map(
+                      ref CacheLineDescription prediction
+        )
+        {
+            if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
+            {
+                if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
+                {
+                    try
+                    {
+                        if (_aheadCount < PredictionLeastAhead)
+                        {
+                            // do prefetch in work load now, sync
+                            Prefetch();
+                        }
+                    }
+                    finally
+                    {
+                        _prefetching = 0;
+                    }
+                }
+            }
+            while (true)
+            {
+                if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
+                {
+                    // cache hit
+                    CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
+                    if (Interlocked.Increment(ref entry->RefCount) > 0)
+                    {
+                        _predictCount[entry - _head] = 0;
+                        _ = Interlocked.Decrement(ref _aheadCount);
+                        _ = Interlocked.Decrement(ref _mapCount[entry - _head]);
+                        nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
+                        return *(Handle*)hPtr;
+                    } else
+                    {
+                        _ = Interlocked.Decrement(ref entry->RefCount);
+                    }
+                }
+                Thread.Sleep(0);
+            }
+        }
+
+        private partial void Prefetch();
+
+    }
+}