//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="L4.cs" // Project: StgSharp // AuthorGroup: Nitload Space // Copyright (c) Nitload Space. All rights reserved. // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation // files (the “Software”), to deal in the Software without restriction, // including without limitation the rights to use, copy, modify, merge, // publish, distribute, sublicense, and/or sell copies of the Software, // and to permit persons to whom the Software is furnished to do so, // subject to the following conditions: // // The above copyright notice and // this permission notice shall be included in all copies // or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED “AS IS”, // WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, // INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. // IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, // ARISING FROM, OUT OF OR IN CONNECTION WITH // THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.Collections; using StgSharp.Common.HighPerformance.ProcessorAbstraction; using StgSharp.Mathematics.Memory; using System.Data; using System.Runtime.InteropServices; using HLSF = StgSharp.Mathematics.Memory.HybridLayerSegregatedFitAllocator; namespace StgSharp.HighPerformance.Memory { public sealed unsafe partial class L4 : IDisposable { private const int CacheLineCount = 8192; private const int PredictionCount = 2048; private const int PredictionLeastAhead = 1024; private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance; private readonly EvictionArray* _eviction; private readonly CacheLineHead* _head; private readonly CacheLinePrediction* _prediction; private bool _disposed; private bool _exhausted; private readonly HLSF _bufferAllocator; private int _activeCount; private int _aheadCount; private readonly int _bindedNuma; private int _maxId; private volatile int _prefetching; private readonly nuint _baseAddress; private readonly SlabAllocator _entryManager; private readonly SwissTable _map; private readonly UnmanagedStack _idMux; public L4(int bindedNuma, nuint bufferSize = 8192 * 8192, int align = 16, int maxCacheLineSize = 8192) { _bindedNuma = bindedNuma; // init order is fallow by align requirement of each component. nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align // malloc all memory at once, and free at once in Dispose. // This is to avoid fragmentation and improve performance. _baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize); nuint ptr = _baseAddress; // init all components with the buffer, and move the buffer pointer forward. _entryManager = SlabAllocator.FromBuffer(); _bufferAllocator = new HLSF(cacheSize, align, maxCacheLineSize, (void*)_baseAddress, static _ => { }); nuint _buffer = _baseAddress; ptr += cacheSize; _head = (CacheLineHead*)ptr; ptr += entrySize; _prediction = (CacheLinePrediction*)ptr; ptr += predictionSize; _map = SwissTable.Create(ptr, _ => { }); ptr += mapSize; _eviction = (EvictionArray*)ptr; ptr += rrpvSize; _idMux = new UnmanagedStack((int*)ptr, CacheLineCount, _ => { }); ptr += stackSize; } public static nuint AllocSize { get; } // public IL4Predict PrefetchPredict { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ContinueOnce(L4CacheLine line) { if (line.Address > 0) { _ = Interlocked.Decrement(ref _aheadCount); } } [System.Diagnostics.CodeAnalysis.SuppressMessage( "Design", "CA1063:Implement IDisposable correctly", Justification = """ Only owns native memory via Numa.Free(); no managed IDisposable fields. Dispose(bool) pattern is unnecessary. """)] public void Dispose() { if (_disposed) { return; } _disposed = true; _map.Dispose(); _idMux.Dispose(); _bufferAllocator.Dispose(); nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align nuint mapSize = SwissTable.RequestedNativeMemorySize; // 16 byte align nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize); GC.SuppressFinalize(this); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public L4CacheLine Map(nuint origin) { if ((_aheadCount < PredictionLeastAhead) && !_exhausted) { if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) { try { if (_aheadCount < PredictionLeastAhead) { // do prefetch in work load now, sync Prefetch(); } } finally { _prefetching = 0; } } } while (true) { if (_map.TryGet(origin, out nuint entryHandle)) { // cache hit CacheLineHead* entry = (CacheLineHead*)entryHandle; if (Interlocked.Increment(ref entry->_refCount) > 0) { (*_eviction)[entry->_id] = 0; _ = Interlocked.Increment(ref _activeCount); _ = Interlocked.Decrement(ref _aheadCount); return new L4CacheLine(entry->_address, ref _activeCount, ref entry->_refCount); } else { _ = Interlocked.Decrement(ref entry->_refCount); } } Thread.Sleep(0); } } private partial void Prefetch(); } public ref struct L4CacheLine : IDisposable { private ref int _activeCount; private ref int _refCount; private nuint _handle; internal L4CacheLine(nuint handle, ref int activeCount, ref int cacheRefCount) { _handle = handle; _refCount = ref cacheRefCount; _activeCount = ref activeCount; } [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] public L4CacheLine() { } public readonly nuint Address => _handle; public void Dispose() { if (_handle == 0) { return; } _handle = 0; _ = Interlocked.Decrement(ref _activeCount); _ = Interlocked.Decrement(ref _refCount); } } }