From 3a2b1234782f893f5b71655289b372ffed8dfd93 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Fri, 9 Oct 2026 18:17:03 +0800 Subject: [PATCH] swiss x86 is briefly ok, ready for test --- src/Nitload.Benchmark/SwissBenchmark.cs | 485 ------------------ src/Nitload.Common/Collections/SwissTable.cs | 125 ----- .../HighPerformance/Memory/L4.CacheLine.cs | 30 +- .../HighPerformance/Memory/L4.Debug.cs | 93 ---- .../HighPerformance/Memory/L4.Evict.cs | 66 --- .../HighPerformance/Memory/L4.Exceptions.cs | 34 -- .../HighPerformance/Memory/L4.Predict.cs | 38 -- .../HighPerformance/Memory/L4.Predictor.cs | 146 ------ .../HighPerformance/Memory/L4.Prefetch.cs | 103 ---- .../HighPerformance/Memory/L4.SwissTable.cs | 217 ++++++++ .../HighPerformance/Memory/L4.Swiss_X86.cs | 200 ++++++++ .../HighPerformance/Memory/L4.Tlsf.cs | 15 +- .../HighPerformance/Memory/L4.cs | 159 +----- test/STgSharpTest/UnitTest1.cs | 17 - test/StgSharpDebug/TestSwiss.cs | 46 -- 15 files changed, 457 insertions(+), 1317 deletions(-) delete mode 100644 src/Nitload.Benchmark/SwissBenchmark.cs delete mode 100644 src/Nitload.Common/Collections/SwissTable.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs delete mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs create mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs create mode 100644 src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs delete mode 100644 test/StgSharpDebug/TestSwiss.cs diff --git a/src/Nitload.Benchmark/SwissBenchmark.cs b/src/Nitload.Benchmark/SwissBenchmark.cs deleted file mode 100644 index c72087cf..00000000 --- a/src/Nitload.Benchmark/SwissBenchmark.cs +++ /dev/null @@ -1,485 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="SwissBenchmark" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using BenchmarkDotNet.Attributes; -using Nitload.Common.Collections; -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; - -namespace Nitload.Benchmark -{ - /// - /// Benchmark comparing SwissTable against under - /// realistic L4 software cache address-mapping workloads. /// L4 scenario - /// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned) - /// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) /// - /// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// - - /// Working set: 512–4096 active cache line mappings /// - /// - [MemoryDiagnoser] - public class SwissTableBench - { - - /// - /// L4 cache line size in bytes (per L4_design.md). - /// - private const int CacheLineSize = 1024; - - /// - /// Simulated heap base address for pointer-like keys. - /// - private const long HeapBase = 0x7FFF_0000_0000L; - - /// - /// Number of lookup operations per Lookup benchmark call. - /// - private const int LookupIterations = 8192; - - /// - /// Fraction of steady-state ops that are cache misses. - /// - private const double MissRate = 0.05; - - /// - /// Number of steady-state operations per call. Each op is either a lookup (hit) or an - /// evict+insert+lookup (miss). - /// - private const int SteadyStateOps = 4096; - - /// - /// Indices into _residentKeys for eviction targets (cold entries, tail region). - /// - private int[] _evictOrder = null!; - - /// - /// Additional keys not in the table, for triggering misses. - /// - private nuint[] _externalKeys = null!; - - // ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ── - - /// - /// Keys for initial table fill (simulates cached addresses). - /// - private nuint[] _residentKeys = null!; - - /// - /// Pre-built steady-state operation script. Each entry: positive index → lookup - /// _residentKeys[idx], negative value → cache miss trigger (evict + insert from - /// _externalKeys). - /// - private int[] _steadyStateScript = null!; - - /// - /// Uniform random lookup sequence (baseline comparison). - /// - private nuint[] _uniformLookups = null!; - - /// - /// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time). - /// - private nuint[] _zipfianLookups = null!; - private Dictionary _dictLookup = null!; - private Dictionary _freshDict = null!; - - // ── Fresh instances for insert/mixed (rebuilt each iteration) ── - - private SwissTable _freshSwiss = null!; - - // ── Pre-filled lookup targets ── - - private SwissTable _swissLookup = null!; - - /// - /// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB / - /// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity. - /// - [Params(512, 1024, 4096)] - public int KeyCount; - - [GlobalCleanup] - public void Cleanup() - { - _dictLookup.Clear(); - _dictLookup.TrimExcess(); - } - - // ════════════════════════════════════════════════════════════ - // Insert — bulk fill with scattered pointer addresses - // Simulates L4 initial cache warm-up phase. - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public void Dictionary_Insert() - { - Dictionary dict = _freshDict; - nuint[] keys = _residentKeys; - for (int i = 0; i < keys.Length; i++) { - dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF); - } - } - - // ════════════════════════════════════════════════════════════ - // Lookup — Uniform (baseline comparison) - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public nuint Dictionary_Lookup_Uniform() - { - nuint sum = 0; - nuint[] keys = _uniformLookups; - Dictionary dict = _dictLookup; - for (int i = 0; i < keys.Length; i++) - { - if (dict.TryGetValue(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // Lookup — Zipfian (L4 hot path, skewed access) - // 20% of cache lines receive 80% of accesses. - // This is the most representative L4 benchmark. - // ════════════════════════════════════════════════════════════ - - [Benchmark(Baseline = true)] - public nuint Dictionary_Lookup_Zipfian() - { - nuint sum = 0; - nuint[] keys = _zipfianLookups; - Dictionary dict = _dictLookup; - for (int i = 0; i < keys.Length; i++) - { - if (dict.TryGetValue(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // SteadyState — the KEY L4 benchmark - // Pre-filled table at capacity, then 4096 operations: - // ~95% are lookups (cache hits) - // ~5% are miss → evict cold entry → insert new entry → lookup new - // This directly simulates L4's main runtime loop. - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public nuint Dictionary_SteadyState() - { - nuint sum = 0; - Dictionary dict = _freshDict; - nuint[] resident = _residentKeys; - nuint[] external = _externalKeys; - int[] script = _steadyStateScript; - int[] evict = _evictOrder; - int evictCursor = 0; - int externalCursor = 0; - - for (int i = 0; i < script.Length; i++) - { - int op = script[i]; - if (op >= 0) - { - // Cache hit: lookup existing entry - if (dict.TryGetValue(resident[op], out nuint value)) - { - sum += value; - } - } else - { - // Cache miss: evict a cold entry, insert a new one, then lookup - int evictIdx = evict[evictCursor % evict.Length]; - evictCursor++; - dict.Remove(resident[evictIdx]); - - int extIdx = externalCursor % external.Length; - externalCursor++; - nuint newKey = external[extIdx]; - nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF); - dict[newKey] = newVal; - - if (dict.TryGetValue(newKey, out nuint v)) { - sum += v; - } - } - } - return sum; - } - - [IterationSetup(Targets = [ - nameof(SwissTable_Insert), - nameof(SwissTable_SteadyState), - nameof(Dictionary_Insert), - nameof(Dictionary_SteadyState) - ])] - public void IterationSetup() - { - _freshSwiss = SwissTable.Create(); - _freshDict = new Dictionary(KeyCount); - - // Pre-fill for steady-state benchmarks - for (int i = 0; i < KeyCount; i++) - { - nuint key = _residentKeys[i]; - nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); - SwissTable.Writer.TrySet(_freshSwiss, key, val); - _freshDict[key] = val; - } - } - - // ════════════════════════════════════════════════════════════ - // Setup / Teardown - // ════════════════════════════════════════════════════════════ - - [GlobalSetup] - public void Setup() - { - Random rng = new Random(42); - - // ── Generate pointer-like keys (HeapBase + random_offset * 1024) ── - _residentKeys = GeneratePointerKeys(KeyCount, rng); - _externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation - - // ── Build lookup sequences ── - _zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng); - _uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng); - - // ── Build steady-state operation script ── - _steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng); - - // ── Build eviction order (cold tail entries first) ── - _evictOrder = new int[KeyCount]; - for (int i = 0; i < KeyCount; i++) { - _evictOrder[i] = i; - } - - // Shuffle so eviction isn't just sequential - Shuffle(_evictOrder, rng); - - // ── Pre-fill lookup tables ── - _swissLookup = SwissTable.Create(); - _dictLookup = new Dictionary(KeyCount); - - for (int i = 0; i < KeyCount; i++) - { - nuint key = _residentKeys[i]; - nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer - SwissTable.Writer.TrySet(_swissLookup, key, val); - _dictLookup[key] = val; - } - } - - [Benchmark] - public void SwissTable_Insert() - { - SwissTable swiss = _freshSwiss; - nuint[] keys = _residentKeys; - for (int i = 0; i < keys.Length; i++) { - SwissTable.Writer.TrySet(swiss, keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF)); - } - } - - [Benchmark] - public nuint SwissTable_Lookup_Uniform() - { - nuint sum = 0; - nuint[] keys = _uniformLookups; - SwissTable swiss = _swissLookup; - for (int i = 0; i < keys.Length; i++) - { - if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - [Benchmark] - public nuint SwissTable_Lookup_Zipfian() - { - nuint sum = 0; - nuint[] keys = _zipfianLookups; - SwissTable swiss = _swissLookup; - for (int i = 0; i < keys.Length; i++) - { - if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - [Benchmark] - public nuint SwissTable_SteadyState() - { - nuint sum = 0; - SwissTable swiss = _freshSwiss; - nuint[] resident = _residentKeys; - nuint[] external = _externalKeys; - int[] script = _steadyStateScript; - int[] evict = _evictOrder; - int evictCursor = 0; - int externalCursor = 0; - - for (int i = 0; i < script.Length; i++) - { - int op = script[i]; - if (op >= 0) - { - // Cache hit: lookup existing entry - if (SwissTable.Reader.TryGet(swiss, resident[op], out nuint value)) - { - sum += value; - } - } else - { - // Cache miss: evict a cold entry, insert a new one, then lookup - int evictIdx = evict[evictCursor % evict.Length]; - evictCursor++; - SwissTable.Writer.TryRemove(swiss, resident[evictIdx], out _); - - int extIdx = externalCursor % external.Length; - externalCursor++; - nuint newKey = external[extIdx]; - nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF); - SwissTable.Writer.TryAdd(swiss, newKey, newVal); - - if (SwissTable.Reader.TryGet(swiss, newKey, out nuint v)) { - sum += v; - } - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // Data generation helpers - // ════════════════════════════════════════════════════════════ - - /// - /// Generate unique pointer-like keys. Each key = HeapBase + - /// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses - /// scattered across a ~1 GB virtual address range. - /// - private static nuint[] GeneratePointerKeys( - int count, - Random rng - ) - { - nuint[] keys = new nuint[count]; - HashSet used = new HashSet(count); - for (int i = 0; i < count; i++) - { - long offset; - do - { - // Random offset within ~1M cache lines (1 GB range) - offset = (long)rng.Next(0, 1 << 20) * CacheLineSize; - } while (!used.Add(offset)); - keys[i] = (nuint)(HeapBase + offset); - } - return keys; - } - - /// - /// Generate a steady-state operation script. Each entry: non-negative value = index into - /// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert → - /// lookup). Lookup indices are Zipfian-skewed. - /// - private static int[] GenerateSteadyStateScript( - int keyCount, - int opCount, - double missRate, - Random rng - ) - { - int hotSetSize = Math.Max(1, keyCount / 5); - int[] script = new int[opCount]; - for (int i = 0; i < opCount; i++) - { - if (rng.NextDouble() < missRate) - { - script[i] = -1; // miss - } else - { - // Zipfian hit - if (rng.NextDouble() < 0.8) - { - script[i] = rng.Next(0, hotSetSize); - } else - { - script[i] = rng.Next(hotSetSize, keyCount); - } - } - } - return script; - } - - /// - /// Generate uniform random lookup sequence (baseline for comparison with Zipfian). - /// - private static nuint[] GenerateUniformLookups( - nuint[] sourceKeys, - int count, - Random rng - ) - { - nuint[] lookups = new nuint[count]; - for (int i = 0; i < count; i++) { - lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)]; - } - return lookups; - } - - /// - /// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys - /// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This - /// models matrix computation where a small panel working set is reused heavily. - /// - private static nuint[] GenerateZipfianLookups( - nuint[] sourceKeys, - int count, - Random rng - ) - { - int hotSetSize = Math.Max(1, sourceKeys.Length / 5); - nuint[] lookups = new nuint[count]; - for (int i = 0; i < count; i++) - { - if (rng.NextDouble() < 0.8) - { - lookups[i] = sourceKeys[rng.Next(0, hotSetSize)]; - } else - { - lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)]; - } - } - return lookups; - } - - /// - /// Fisher-Yates shuffle. - /// - private static void Shuffle( - int[] array, - Random rng - ) - { - for (int i = array.Length - 1; i > 0; i--) - { - int j = rng.Next(0, i + 1); - (array[i], array[j]) = (array[j], array[i]); - } - } - - } -} diff --git a/src/Nitload.Common/Collections/SwissTable.cs b/src/Nitload.Common/Collections/SwissTable.cs deleted file mode 100644 index 3f9c3eb6..00000000 --- a/src/Nitload.Common/Collections/SwissTable.cs +++ /dev/null @@ -1,125 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="SwissTable" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.HighPerformance.ProcessorAbstraction; -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Nitload.Common.Collections -{ - public sealed unsafe partial class SwissTable - { - - private const int BucketSize = 16; - private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation - private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket - - private const int GroupCount = 1 << EntryMaskWidth; - public const int MaxCapacity = GroupCount * BucketSize; // 256 entries per bucket, 1024 buckets - - public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf() + (nuint)Unsafe.SizeOf(); - - private byte* _controlBytes; - - private SwissTable( - byte* buffer - ) - { - _controlBytes = buffer; - } - - private M128* KvpArray => (M128*)(_controlBytes + MaxCapacity); - - private static void Initialize( - byte* buffer - ) - { - Unsafe.InitBlock(buffer, 0b10000000, BucketSize); - } - - private unsafe struct Bucket - { - - public BucketControl Control; - public BucketKeyStorage Keys; - public BucketValueStorage Values; - - } - - public static class Reader - { - - public static bool TryGet( - SwissTable table, - nuint key, - out nuint value - ) - { - throw new NotImplementedException("SwissTable reader is awaiting a redesigned storage and lookup implementation."); - } - - } - - public static class Writer - { - - public static bool TryAdd( - SwissTable table, - nuint key, - nuint value - ) - { - throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and insertion implementation."); - } - - public static bool TryRemove( - SwissTable table, - nuint key, - out nuint value - ) - { - throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and removal implementation."); - } - - public static bool TrySet( - SwissTable table, - nuint key, - nuint value - ) - { - throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and update implementation."); - } - - } - - [InlineArray(BucketSize)] - private struct BucketControl - { - - public byte Control; - - } - - [InlineArray(BucketSize)] - private struct BucketKeyStorage - { - - public nuint value; - - } - - [InlineArray(BucketSize)] - private struct BucketValueStorage - { - - public nuint value; - - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs index 575bd318..7c605e60 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -5,7 +5,6 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.Common.HighPerformance.ProcessorAbstraction; using System.Runtime.InteropServices; namespace Nitload.Common.HighPerformance.Memory @@ -23,15 +22,11 @@ namespace Nitload.Common.HighPerformance.Memory } [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)] - private unsafe struct CacheLineMetadata + private struct CacheLineMetadata { internal const uint MaxBufferSize = 0xFFFFFFFF - 64U; - [FieldOffset(0)] private readonly M256 FillingMask; - - [FieldOffset(24)] internal nuint Origin; - public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); #region tlsf linked list @@ -50,32 +45,17 @@ namespace Nitload.Common.HighPerformance.Memory #endregion - #region mask - [FieldOffset(16)] internal AllocationState State; - [FieldOffset(17)] internal byte ReversedProfile; - [FieldOffset(18)] internal ushort CustomizeProfile; - [FieldOffset(20)] internal int RefCount; - #endregion } #pragma warning disable IDE0032 - [StructLayout(LayoutKind.Explicit)] - public unsafe struct Handle : IDisposable + [StructLayout(LayoutKind.Explicit, Size = 16)] + public struct Handle : IDisposable { - [FieldOffset(8)] private readonly CacheLineMetadata* _head; [FieldOffset(0)] private nuint _handle; - - private Handle( - nuint handle, - CacheLineMetadata* cacheRefCount - ) - { - _handle = handle; - _head = cacheRefCount; - } + [FieldOffset(8)] private nuint _reserved; [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] public Handle() { } @@ -88,7 +68,7 @@ namespace Nitload.Common.HighPerformance.Memory return; } _handle = 0; - _ = Interlocked.Decrement(ref _head->RefCount); + _reserved = 0; } } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs deleted file mode 100644 index fff7dc56..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs +++ /dev/null @@ -1,93 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Debug" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - /**/ - public partial class L4 - { - - public void DebugAgeAll() - { - // AgeAll(); - } - - public bool DebugContains( - nuint origin - ) - { - return SwissTable.Reader.TryGet(_map, origin, out _); - } - - public int DebugEvict( - int count = 1 - ) - { - return Evict(count); - } - - public unsafe DebugSnapshot GetDebugSnapshot() - { - throw new NotImplementedException(); - /* - int maxId = _maxId; - byte[] rrpv = new byte[maxId]; - byte* p = (byte*)_eviction; - for (int i = 0; i < maxId; i++) { - rrpv[i] = p[i]; - } - return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv); - /**/ - } - - public readonly struct DebugSnapshot - { - - /// - /// RRPV value for each line（0~3）, length == MaxId - /// - public readonly byte[] RrpvSnapshot; - /// - /// Count of activated cache line handle - /// - public readonly int ActiveCount; - - /// - /// Count of predictor ahead of current map position（aheadCount） - /// - public readonly int AheadCount; - - /// - /// Max ID of the line being using or once used（maxId） - /// - public readonly int MaxId; - - internal DebugSnapshot( - int activeCount, - int aheadCount, - int maxId, - byte[] rrpv - ) - { - ActiveCount = activeCount; - AheadCount = aheadCount; - MaxId = maxId; - RrpvSnapshot = rrpv; - } - - } - - } - /**/ -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs deleted file mode 100644 index 82cf0c26..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs +++ /dev/null @@ -1,66 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Evict" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; - -namespace Nitload.Common.HighPerformance.Memory -{ - public partial class L4 - { - - /// - /// - /// - /// - /// - /// - /// - /// Count of lines not evicted - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe int Evict( - int count = 1 - ) - { - count = (count < 0) ? 0 : count; - int* pPtr = _predictCount, mPtr = _mapCount; - - - int remain = count; - for (int i = 0; i < CacheLineCount; i++) - { - if (pPtr[i] > mPtr[i]) - { - continue; - } - remain -= TryEvictAt(i) ? 1 : 0; - } - return remain; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe bool TryEvictAt( - int index - ) - { - CacheLineMetadata* head = &_head[index]; - if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) - { - // EvictLine(i + index); - _ = SwissTable.Writer.TryRemove(_map, head->Origin, out _); - IL4Predict predict = CurrentPredict; - predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask)); - _bufferAllocator.Free(index); - _mapCount[index] = 0; - _predictCount[index] = int.MaxValue; - return true; - } - return false; - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs deleted file mode 100644 index de4efadb..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Exceptions" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -namespace Nitload.Common.HighPerformance.Memory -{ -#pragma warning disable CA1032 // Implement standard exception constructors - - public sealed class L4PredictUnregisteredException : Exception - { - - public L4PredictUnregisteredException( - int id - ) - : base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { } - - } - - public sealed class L4PredictConflictException : Exception - { - - public L4PredictConflictException( - nuint basePtr, - ushort policy, - int current - ) - : base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { } - - } -#pragma warning restore CA1032 // Implement standard exception constructors -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs deleted file mode 100644 index d30b411c..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs +++ /dev/null @@ -1,38 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Predict" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using Nitload.Common.HighPerformance.Memory; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - public partial class L4 - { - - private IL4Predict CurrentPredict { get; set; } - - public void ResetPredict( - IL4Predict predict - ) - { - if (CurrentPredict is null) - { - CurrentPredict = predict; - return; - } - - // TODO: write back all and register new predictor - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs deleted file mode 100644 index 4344483c..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs +++ /dev/null @@ -1,146 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Predictor" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - [Flags] - public enum PredictResult - { - - Success = 1, - Exhausted = 2, - - } - - public interface IL4Predict - { - - PredictResult Predict( - Span node, - out int count - ); - - /// - /// Materializes the cache line identified by into . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Destination cache line buffer. - /// - void Prefetch( - nuint origin, - uint policy, - Span cache - ); - - /// - /// Writes the cache line in back to . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Source cache line buffer. - /// - void WriteBack( - nuint origin, - uint policy, - ReadOnlySpan cache - ); - - } - - [StructLayout(LayoutKind.Explicit)] - public struct CacheLineDescription : IEquatable - { - - public override readonly bool Equals( - object? obj - ) - { - return (obj is CacheLineDescription prediction) && - Address.Equals(prediction.Address) && - (MapPolicy == prediction.MapPolicy); - } - - public readonly bool Equals( - CacheLineDescription other - ) - { - return - (Address == other.Address) && (MapPolicy == other.MapPolicy); - } - - public override readonly int GetHashCode() - { - return (int)ComposePredictionKey64(Address, MapPolicy); - } - - public readonly nuint GetHashCodeLong() - { - return ComposePredictionKey64(Address, MapPolicy); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static nuint ComposePredictionKey64( - nuint address, - ushort mapPolicy - ) - { - nuint x = address; - nuint y = mapPolicy; - unchecked - { - nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL)); - mixed ^= mixed >> 30; - mixed *= (nuint)0xbf58476d1ce4e5b9UL; - mixed ^= mixed >> 27; - mixed *= (nuint)0x94d049bb133111ebUL; - mixed ^= mixed >> 31; - return mixed; - } - } - - public static bool operator !=( - CacheLineDescription left, - CacheLineDescription right - ) - { - return !(left == right); - } - public static bool operator ==( - CacheLineDescription left, - CacheLineDescription right - ) - { - return left.Equals(right); - } - -#pragma warning disable CA1051 - [FieldOffset(0)]public nuint Address; - [FieldOffset(8)]public uint Size; - [FieldOffset(12)]public ushort MapPolicy; - [FieldOffset(14)]internal short Predictor; -#pragma warning restore CA1051 - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs deleted file mode 100644 index 0a87f428..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs +++ /dev/null @@ -1,103 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Prefetch" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using System.ComponentModel; -using System.Numerics; - -namespace Nitload.Common.HighPerformance.Memory -{ - public unsafe partial class L4 - { - - private partial void Prefetch() - { - if (CurrentPredict is null) { - return; - } - IL4Predict predict = CurrentPredict; - Span span = new(_prediction, PredictionCount); - PredictResult result = predict.Predict(span, out int actualCount); - _exhausted = (result & PredictResult.Exhausted) != 0; - if (actualCount <= 0) - { - _exhausted = true; - return; - } - int missIndex = 0; - for (int i = 0; i < actualCount; i++) - { - ref CacheLineDescription handle = ref span[i]; - nuint address = handle.Address; - if (!SwissTable.Reader.TryGet(_map, address, out _)) - { - // miss and record - span[missIndex] = handle; - missIndex++; - } - } - - // now prefetch the misses - for (int i = 0; i < missIndex; i++) - { - CacheLineMetadata* head; - ref CacheLineDescription pHandle = ref span[i]; - nuint address = pHandle.Address; - int index; - if (SwissTable.Reader.TryGet(_map, address, out nuint handle)) - { - head = (CacheLineMetadata*)handle; - index = (int)(head - _head); - _predictCount[index] = 0; // reset eviction counter - - #region update predictor belonging - - if (_predictCount[index] > short.MaxValue / 2) - { - int predictCount = Volatile.Read(ref _predictCount[index]); - int mapCount = Volatile.Read(ref _mapCount[index]); - int decrement = (int.Min(predictCount, mapCount) + 1) / 2; - _ = Interlocked.Add(ref _predictCount[index], -decrement); - _ = Interlocked.Add(ref _mapCount[index], -decrement); - } - _predictCount[index]++; - - #endregion - - continue; - } - - int retryCount = 0, remain = missIndex - i; - const int maxRetry = 1024; - while (!_bufferAllocator.TryAlloc(pHandle.Size, out index)) - { - remain = Evict(remain); - retryCount++; - if (retryCount > maxRetry) { - throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); - } - if (retryCount > maxRetry / 2) { - Thread.Sleep(1); - } - } - head = _head + index; - head->Origin = address; - head->CustomizeProfile = pHandle.MapPolicy; - head->RefCount = 0; - - _predictCount[index] = 1; - _mapCount[index] = 1; - - // pre fetch them here - predict.Prefetch(head->Origin, head->CustomizeProfile, new Span((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask)); - _ = SwissTable.Writer.TrySet(_map, address, (nuint)head); - } - _ = Interlocked.Add(ref _aheadCount, actualCount); - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs new file mode 100644 index 00000000..f87d40bd --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs @@ -0,0 +1,217 @@ +// ----------------------------------------------------------------------------- +// file="L4.SwissTable" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Nitload.Common.HighPerformance.Memory +{ + public sealed unsafe partial class L4 + { + + private const int BucketCapacity = 16; + private const uint CrcMagic = 0x7f1c9961; + + private const int GroupCount = 1 << GroupCountLog; + private const int GroupCountLog = 10; + private const int IdealCount = GroupCount * BucketCapacity;//16384 + private const int OffsetOfBucket = IdealCount; + + private static readonly int TableBufferSize = + IdealCount + (GroupCount * Unsafe.SizeOf()); + internal static readonly int BufferSize = TableBufferSize * 2; + + private enum SwissControl : byte + { + + Empty = 0b10000000, + Deleted = 0b10000001 + + } + + [StructLayout(LayoutKind.Sequential)] + internal struct AddressSource + { + + public ulong Address; + public ulong Description; + + public override readonly bool Equals( + object obj + ) + { + return obj is AddressSource source && this == source; + } + + public override readonly int GetHashCode() + { + return HashCode.Combine(Address, Description); + } + + public static bool operator !=( + AddressSource left, + AddressSource right + ) + { + return !(left == right); + } + + public static bool operator ==( + AddressSource left, + AddressSource right + ) + { + return left.Address == right.Address && left.Description == right.Description; + } + + } + + [InlineArray(BucketCapacity)] + internal struct KeyArray + { + + private AddressSource _element0; + + } + + internal struct SwissBucket + { + + public KeyArray Keys; + public fixed ulong Values[BucketCapacity]; + + } + + internal sealed unsafe class SwissTable : IDisposable + { + + private byte* _buffer_read; + private byte* _buffer_write; + private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion); + + private SwissTable(byte* buffers) + { + _buffer_read = buffers; + _buffer_write = buffers + TableBufferSize; + InitializeBuffer(_buffer_read); + InitializeBuffer(_buffer_write); + } + + internal static SwissTable Create(byte* buffers) + { + ArgumentNullException.ThrowIfNull(buffers); + return new SwissTable(buffers); + } + + /// + /// Reads the published buffer. Callers must prevent a swap while the read is in progress. + /// + public bool TryGet(AddressSource key, out ulong value) + { + _synchronization.EnterReadLock(); + try + { + return SwissReaderX86.TryGet(_buffer_read, key, out value); + } + finally + { + _synchronization.ExitReadLock(); + } + } + + /// + /// Writes to the inactive buffer. Calls must be serialized by the caller. + /// + public bool TryAddOrSet(AddressSource key, ulong value) + { + _synchronization.EnterWriteLock(); + try + { + return SwissWriterX86.TryAddOrSet(_buffer_write, key, value); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Writes to the inactive buffer. Calls must be serialized by the caller. + /// + public bool TryRemove(AddressSource key, out ulong value) + { + _synchronization.EnterWriteLock(); + try + { + return SwissWriterX86.TryRemove(_buffer_write, key, out value); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced. + /// + public void PrepareWriteBuffer() + { + _synchronization.EnterWriteLock(); + try + { + InitializeBuffer(_buffer_write); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Publishes the write buffer. The caller must provide the phase barrier around this operation. + /// + public void SwapBuffers() + { + _synchronization.EnterWriteLock(); + try + { + byte* temporary = _buffer_read; + _buffer_read = _buffer_write; + _buffer_write = temporary; + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + public void Dispose() + { + _synchronization.EnterWriteLock(); + try + { + _buffer_read = null; + _buffer_write = null; + } + finally + { + _synchronization.ExitWriteLock(); + } + _synchronization.Dispose(); + } + + private static void InitializeBuffer(byte* buffer) + { + Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount); + } + + } + + } +} \ No newline at end of file diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs new file mode 100644 index 00000000..a19f7944 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs @@ -0,0 +1,200 @@ +// ----------------------------------------------------------------------------- +// file="L4.Swiss_X86" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System.Numerics; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace Nitload.Common.HighPerformance.Memory +{ + public partial class L4 + { + + private readonly unsafe struct SwissWriterX86 + { + + public static bool TryAddOrSet( + byte* buffer, + AddressSource key, + ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + + uint first_tomb_group = 0; + int first_tomb_idx = -1; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128* control_bytes_ptr = control_bytes + begin; + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr); + + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted); + + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + Vector128 cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128); + + int hash_mask = Sse2.MoveMask(cmp_hash); + int empty_mask = Sse2.MoveMask(cmp_empty); + int tomb_mask = Sse2.MoveMask(cmp_tomb); + + if (tomb_mask != 0 && first_tomb_idx < 0) + { + int index = BitOperations.TrailingZeroCount(tomb_mask); + first_tomb_group = begin; + first_tomb_idx = index; + } + while (hash_mask != 0) + { + int index = BitOperations.TrailingZeroCount(hash_mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + bucket->Values[index] = value; + return true; + } + hash_mask &= ~(1 << index); + } + while (empty_mask != 0) + { + int index = BitOperations.TrailingZeroCount(empty_mask); + if (first_tomb_idx < 0) + { + SwissBucket* bucket = &buckets[begin]; + bucket->Keys[index] = key; + bucket->Values[index] = value; + ((byte*)control_bytes_ptr)[index] = target_hash; + } else + { + SwissBucket* bucket = &buckets[first_tomb_group]; + bucket->Keys[first_tomb_idx] = key; + bucket->Values[first_tomb_idx] = value; + ((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash; + } + return true; + } + + begin = (begin + 1) & (GroupCount - 1); + } + if (first_tomb_idx >= 0) + { + SwissBucket* bucket = &buckets[first_tomb_group]; + bucket->Keys[first_tomb_idx] = key; + bucket->Values[first_tomb_idx] = value; + ((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash; + return true; + } + value = 0; + return false; + } + + public static bool TryRemove( + byte* buffer, + AddressSource key, + out ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin)); + + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + + int mask = Sse2.MoveMask(cmp_hash); + + while (mask != 0) + { + int index = BitOperations.TrailingZeroCount(mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + ((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted; + value = bucket->Values[index]; + return true; + } + mask &= ~(1 << index); + } + + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + int empty_mask = Sse2.MoveMask(cmp_empty); + if (empty_mask != 0) + { + break; + } + begin = (begin + 1) & (GroupCount - 1); + } + value = 0; + return false; + } + + } + + private readonly unsafe struct SwissReaderX86 + { + + public static bool TryGet( + byte* buffer, + AddressSource key, + out ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin)); + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + int mask = Sse2.MoveMask(cmp_hash); + while (mask != 0) + { + int index = BitOperations.TrailingZeroCount(mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + value = bucket->Values[index]; + return true; + } + mask &= ~(1 << index); + } + + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + if (Sse2.MoveMask(cmp_empty) != 0) + break; + + begin = (begin + 1) & (GroupCount - 1); + } + value = 0; + return false; + } + + } + + } +} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs index 1b9a873a..40a8a396 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs @@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory public partial class L4 { - private unsafe class TLSF + private unsafe struct TLSF { public const int MaxCacheLineCount = 8192; - public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf; - - private readonly short[] _bucket; + public static readonly nuint RequestedAllocation = + ((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf; private readonly CacheLineMetadata* _spareMemory; private readonly int _align; private readonly int _maxLevel; private int _metadataEnd; private readonly Ptr64 _basePtr; + + private fixed short _bucket[64]; private readonly uint _maxAllocSize; private readonly uint _totalSize; @@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory _maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U)); _maxLevel = int.MaxValue; _maxLevel = GetLevel(_maxAllocSize); - _bucket = new short[GetLevel(_maxAllocSize) + 2]; + + // _bucket = new short[GetLevel(_maxAllocSize) + 2]; + _bucket[63] = (short)(GetLevel(_maxAllocSize) + 2); _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _metadataEnd = 0; @@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory idx = candidate->NextLevel; } } - for (int i = level + 1; i < _bucket.Length; i++) + for (int i = level + 1; i < _bucket[63]; i++) { int idx = _bucket[i]; if (idx != 0) diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.cs b/src/Nitload.Common/HighPerformance/Memory/L4.cs index e91f65ef..e0c30f3f 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.cs @@ -5,49 +5,20 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.Common.Collections; -using Nitload.Common.HighPerformance.Memory; using Nitload.Common.HighPerformance.ProcessorAbstraction; -using System.Data; -using System.Runtime.InteropServices; namespace Nitload.Common.HighPerformance.Memory { public sealed unsafe partial class L4 : IDisposable { - private const int CacheLineCount = 8192; private const nuint PageSize = 4096; - private const int PredictionCount = 2048; - private const int PredictionLeastAhead = 1024; - /// - /// Array of metadata for each cache line, also used as metadata for tlsf allocator. - /// - private readonly CacheLineMetadata* _head; - /// - /// Count of cache lines mapped for each prediction, used for eviction policy. - /// - private readonly int* _mapCount; - /// - /// Count of times each cache line is predicted, used for eviction policy. - /// - private readonly int* _predictCount; - /// - /// Original memory view for prediction span. - /// - private readonly CacheLineDescription* _prediction; private bool _disposed; - private bool _exhausted; - - private int _aheadCount; - private readonly int _bindedNuma; - private volatile int _prefetching; private readonly nuint _baseAddress; - private readonly SwissTable _map; - - private readonly TLSF _bufferAllocator; + private SwissTable _addressMap; + private TLSF _bufferAllocator; public L4( int bindedNuma, @@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize); ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize); _bindedNuma = bindedNuma; - - // init order is fallow by align requirement of each component. - nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align - nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align - nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align - nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align - nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size + nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); ByteCapacity = normalizedBufferSize; - - AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize; - - // malloc all memory at once, and free at once in Dispose. - // This is to avoid fragmentation and improve performance. + nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation; + AllocSize = swissBufferOffset + (nuint)BufferSize; _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize); - nuint ptr = _baseAddress; - - // init all components with the buffer, and move the buffer pointer forward. - nuint _buffer = _baseAddress; _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress); - _buffer += normalizedBufferSize; - _head = (CacheLineMetadata*)ptr; - ptr += headSize; - _prediction = (CacheLineDescription*)ptr; - ptr += predictionSize; - _map = SwissTable.Create(ptr, _ => { }); - ptr += mapCountSize; - _predictCount = (int*)ptr; - ptr += predictionCountSize; - _mapCount = (int*)ptr; - ptr += predictionCountSize; - - M512* mPtr = (M512*)_head; - for (uint i = 1; i < CacheLineCount; i++) { - mPtr->BroadCastFrom(0); - } + _addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset)); } public nuint ByteCapacity { get; init; } public nuint AllocSize { get; init; } - // public IL4Predict PrefetchPredict { get; set; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ContinueOnce( - Handle line - ) - { - if (line.Address > 0) { - _ = Interlocked.Decrement(ref _aheadCount); - } - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Design", - "CA1063:Implement IDisposable correctly", - Justification = """ - Only owns native memory via Numa.Free(); - no managed IDisposable fields. - Dispose(bool) pattern is unnecessary. - """)] public void Dispose() { if (_disposed) { return; } _disposed = true; - _map.Dispose(); -#pragma warning disable CA1816 - GC.SuppressFinalize(_bufferAllocator); -#pragma warning restore CA1816 - - - // init order is fallow by align requirement of each component. + _addressMap.Dispose(); Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize); GC.SuppressFinalize(this); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Handle Map( - ref CacheLineDescription prediction - ) + internal void Collect() { - if ((_aheadCount < PredictionLeastAhead) && !_exhausted) - { - if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) - { - try - { - if (_aheadCount < PredictionLeastAhead) - { - // do prefetch in work load now, sync - Prefetch(); - } - } - finally - { - _prefetching = 0; - } - } - } - while (true) - { - if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle)) - { - // cache hit - CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle; - if (Interlocked.Increment(ref entry->RefCount) > 0) - { - _predictCount[entry - _head] = 0; - _ = Interlocked.Decrement(ref _aheadCount); - _ = Interlocked.Decrement(ref _mapCount[entry - _head]); - nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry }; - return *(Handle*)hPtr; - } else - { - _ = Interlocked.Decrement(ref entry->RefCount); - } - } - Thread.Sleep(0); - } + ObjectDisposedException.ThrowIf(_disposed, this); + _bufferAllocator.Collect(); } - private partial void Prefetch(); + internal void Free( + int handle + ) + { + ObjectDisposedException.ThrowIf(_disposed, this); + _bufferAllocator.Free(handle); + } + + internal bool TryAllocate( + uint size, + out int handle + ) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return _bufferAllocator.TryAlloc(size, out handle); + } } } diff --git a/test/STgSharpTest/UnitTest1.cs b/test/STgSharpTest/UnitTest1.cs index 998f444c..897eeaec 100644 --- a/test/STgSharpTest/UnitTest1.cs +++ b/test/STgSharpTest/UnitTest1.cs @@ -32,7 +32,6 @@ using StgSharp; using StgSharp.Collections; using StgSharp.Script; using StgSharp.Script.Express; -using Nitload.Common.Collections; //using StgSharp.Model; @@ -47,22 +46,6 @@ namespace STgSharpTest { string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; - [TestMethod] - public void SwissTableOperationsAreNotImplemented() - { - using SwissTable table = SwissTable.Create(); - nuint key = 17; - - Assert.ThrowsException( - () => SwissTable.Reader.TryGet(table, key, out _)); - Assert.ThrowsException( - () => SwissTable.Writer.TryAdd(table, key, 100)); - Assert.ThrowsException( - () => SwissTable.Writer.TrySet(table, key, 100)); - Assert.ThrowsException( - () => SwissTable.Writer.TryRemove(table, key, out _)); - } - [TestMethod] public void RegexTest() { diff --git a/test/StgSharpDebug/TestSwiss.cs b/test/StgSharpDebug/TestSwiss.cs deleted file mode 100644 index 3997e1ff..00000000 --- a/test/StgSharpDebug/TestSwiss.cs +++ /dev/null @@ -1,46 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="TestSwiss" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload -{ - internal static partial class Program - { - - public static void TestSwiss() - { - SwissTable swiss = SwissTable.Create(); - for (nuint i = 0; i < 5000; i++) - { - Console.Write($"Set {i}: "); - _ = SwissTable.Writer.TrySet(swiss, i, i * 2); - Console.WriteLine($"Set a kvp of ({i},{i * 2})"); - } - Random rand = new Random(); - for (int i = 0; i < 100; i++) - { - Console.Write($"Get {i}: "); - nuint key = (nuint)rand.Next(4000, 5000); - if (SwissTable.Reader.TryGet(swiss, key, out nuint value)) - { - Console.WriteLine($"Success in get a kvp of ({key},{value})"); - } else - { - Console.WriteLine($"Failed to get a kvp of ({key},?)"); - } - } - } - - } -}