stgsharp/StgSharp.Common/HighPerformance/Memory/L4.cs

//-----------------------------------------------------------------------
// -----------------------------------------------------------------------
// file="L4.cs"
// Project: StgSharp
// AuthorGroup: Nitload Space
// Copyright (c) Nitload Space. All rights reserved.
//     
// Permission is hereby granted, free of charge, to any person 
// obtaining a copy of this software and associated documentation 
// files (the “Software”), to deal in the Software without restriction, 
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software, 
// and to permit persons to whom the Software is furnished to do so, 
// subject to the following conditions:
//     
// The above copyright notice and 
// this permission notice shall be included in all copies 
// or substantial portions of the Software.
//     
// THE SOFTWARE IS PROVIDED “AS IS”, 
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 
// ARISING FROM, OUT OF OR IN CONNECTION WITH 
// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//     
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.Common.HighPerformance.ProcessorAbstraction;
using StgSharp.Mathematics.Memory;
using System.Data;
using System.Runtime.InteropServices;

using HLSF = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator;

namespace StgSharp.HighPerformance.Memory
{
    public sealed unsafe partial class L4 : IDisposable
    {

        private const int CacheLineCount = 8192;
        private const int PredictionCount = 2048;
        private const int PredictionLeastAhead = 1024;

        private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance;
        private readonly EvictionArray* _eviction;
        private readonly CacheLineHead* _head;
        private readonly CacheLinePrediction* _prediction;
        private bool _disposed;
        private bool _exhausted;

        private readonly HLSF _bufferAllocator;
        private int _activeCount;

        private int _aheadCount;

        private readonly int _bindedNuma;
        private int _maxId;
        private volatile int _prefetching;
        private readonly nuint _baseAddress;
        private readonly SlabAllocator<HLSF.Entry> _entryManager;
        private readonly SwissTable _map;
        private readonly UnmanagedStack<int> _idMux;

        public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, int maxCacheLineSize = 8192)
        {
            _bindedNuma = bindedNuma;

            // init order is fallow by align requirement of each component.
            nuint cacheSize = ((nuint)Unsafe.SizeOf<HLSF.Entry>()) * CacheLineCount;   // SIMDID, no less than 16 byte bit
            nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount;   // 16 byte align
            nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount;   //16 byte align
            nuint mapSize = SwissTable.RequestedNativeMemorySize;     // 16 byte align
            nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>();           // 16 byte align
            nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount;                      // int align

            // malloc all memory at once, and free at once in Dispose.
            // This is to avoid fragmentation and improve performance.
            _baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
            nuint ptr = _baseAddress;

            // init all components with the buffer, and move the buffer pointer forward.
            _entryManager = SlabAllocator.FromBuffer();

            _bufferAllocator = new HLSF(cacheSize, align, maxCacheLineSize, (void*)_baseAddress, static _ => { });
            nuint _buffer = _baseAddress;
            ptr += cacheSize;
            _head = (CacheLineHead*)ptr;
            ptr += entrySize;
            _prediction = (CacheLinePrediction*)ptr;
            ptr += predictionSize;
            _map = SwissTable.Create(ptr, _ => { });
            ptr += mapSize;
            _eviction = (EvictionArray*)ptr;
            ptr += rrpvSize;
            _idMux = new UnmanagedStack<int>((int*)ptr, CacheLineCount, _ => { });
            ptr += stackSize;
        }

        public static nuint AllocSize { get; }

        // public IL4Predict PrefetchPredict { get; set; }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public void ContinueOnce(L4CacheLine line)
        {
            if (line.Address > 0) {
                _ = Interlocked.Decrement(ref _aheadCount);
            }
        }

        [System.Diagnostics.CodeAnalysis.SuppressMessage(
                "Design",
                "CA1063:Implement IDisposable correctly",
                Justification = """
                    Only owns native memory via Numa.Free();
                    no managed IDisposable fields.
                    Dispose(bool) pattern is unnecessary.
                    """)]
        public void Dispose()
        {
            if (_disposed) {
                return;
            }
            _disposed = true;
            _map.Dispose();
            _idMux.Dispose();
            _bufferAllocator.Dispose();


            nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount;   // SIMDID, no less than 16 byte bit
            nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount;   // 16 byte align
            nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount;   //16 byte align
            nuint mapSize = SwissTable.RequestedNativeMemorySize;     // 16 byte align
            nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>();           // 16 byte align
            nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount;                      // int align
            Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize);
            GC.SuppressFinalize(this);
        }

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        public L4CacheLine Map(nuint origin)
        {
            if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
            {
                if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
                {
                    try
                    {
                        if (_aheadCount < PredictionLeastAhead)
                        {
                            // do prefetch in work load now, sync
                            Prefetch();
                        }
                    }
                    finally
                    {
                        _prefetching = 0;
                    }
                }
            }
            while (true)
            {
                if (_map.TryGet(origin, out nuint entryHandle))
                {
                    // cache hit
                    CacheLineHead* entry = (CacheLineHead*)entryHandle;
                    if (Interlocked.Increment(ref entry->_refCount) > 0)
                    {
                        (*_eviction)[entry->_id] = 0;
                        _ = Interlocked.Increment(ref _activeCount);
                        _ = Interlocked.Decrement(ref _aheadCount);
                        return new L4CacheLine(entry->_address, ref _activeCount, ref entry->_refCount);
                    } else
                    {
                        _ = Interlocked.Decrement(ref entry->_refCount);
                    }
                }
                Thread.Sleep(0);
            }
        }

        private partial void Prefetch();

    }

    public ref struct L4CacheLine : IDisposable
    {

        private ref int _activeCount;
        private ref int _refCount;
        private nuint _handle;

        internal L4CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount)
        {
            _handle = handle;
            _refCount = ref cacheRefCount;
            _activeCount = ref activeCount;
        }

        [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
        public L4CacheLine() { }

        public readonly nuint Address => _handle;

        public void Dispose()
        {
            if (_handle == 0) {
                return;
            }
            _handle = 0;
            _ = Interlocked.Decrement(ref _activeCount);
            _ = Interlocked.Decrement(ref _refCount);
        }

    }
}