diff --git a/src/Nitload.Benchmark/SwissBenchmark.cs b/src/Nitload.Benchmark/SwissBenchmark.cs deleted file mode 100644 index 02596b8d..00000000 --- a/src/Nitload.Benchmark/SwissBenchmark.cs +++ /dev/null @@ -1,485 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="SwissBenchmark" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using BenchmarkDotNet.Attributes; -using Nitload.Common.Collections; -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; - -namespace Nitload.Benchmark -{ - /// - /// Benchmark comparing SwissTable against under - /// realistic L4 software cache address-mapping workloads. /// L4 scenario - /// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned) - /// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) /// - /// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// - - /// Working set: 512–4096 active cache line mappings /// - /// - [MemoryDiagnoser] - public class SwissTableBench - { - - /// - /// L4 cache line size in bytes (per L4_design.md). - /// - private const int CacheLineSize = 1024; - - /// - /// Simulated heap base address for pointer-like keys. - /// - private const long HeapBase = 0x7FFF_0000_0000L; - - /// - /// Number of lookup operations per Lookup benchmark call. - /// - private const int LookupIterations = 8192; - - /// - /// Fraction of steady-state ops that are cache misses. - /// - private const double MissRate = 0.05; - - /// - /// Number of steady-state operations per call. Each op is either a lookup (hit) or an - /// evict+insert+lookup (miss). - /// - private const int SteadyStateOps = 4096; - - /// - /// Indices into _residentKeys for eviction targets (cold entries, tail region). - /// - private int[] _evictOrder = null!; - - /// - /// Additional keys not in the table, for triggering misses. - /// - private nuint[] _externalKeys = null!; - - // ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ── - - /// - /// Keys for initial table fill (simulates cached addresses). - /// - private nuint[] _residentKeys = null!; - - /// - /// Pre-built steady-state operation script. Each entry: positive index → lookup - /// _residentKeys[idx], negative value → cache miss trigger (evict + insert from - /// _externalKeys). - /// - private int[] _steadyStateScript = null!; - - /// - /// Uniform random lookup sequence (baseline comparison). - /// - private nuint[] _uniformLookups = null!; - - /// - /// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time). - /// - private nuint[] _zipfianLookups = null!; - private Dictionary _dictLookup = null!; - private Dictionary _freshDict = null!; - - // ── Fresh instances for insert/mixed (rebuilt each iteration) ── - - private SwissTable _freshSwiss = null!; - - // ── Pre-filled lookup targets ── - - private SwissTable _swissLookup = null!; - - /// - /// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB / - /// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity. - /// - [Params(512, 1024, 4096)] - public int KeyCount; - - [GlobalCleanup] - public void Cleanup() - { - _dictLookup.Clear(); - _dictLookup.TrimExcess(); - } - - // ════════════════════════════════════════════════════════════ - // Insert — bulk fill with scattered pointer addresses - // Simulates L4 initial cache warm-up phase. - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public void Dictionary_Insert() - { - Dictionary dict = _freshDict; - nuint[] keys = _residentKeys; - for (int i = 0; i < keys.Length; i++) { - dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF); - } - } - - // ════════════════════════════════════════════════════════════ - // Lookup — Uniform (baseline comparison) - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public nuint Dictionary_Lookup_Uniform() - { - nuint sum = 0; - nuint[] keys = _uniformLookups; - Dictionary dict = _dictLookup; - for (int i = 0; i < keys.Length; i++) - { - if (dict.TryGetValue(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // Lookup — Zipfian (L4 hot path, skewed access) - // 20% of cache lines receive 80% of accesses. - // This is the most representative L4 benchmark. - // ════════════════════════════════════════════════════════════ - - [Benchmark(Baseline = true)] - public nuint Dictionary_Lookup_Zipfian() - { - nuint sum = 0; - nuint[] keys = _zipfianLookups; - Dictionary dict = _dictLookup; - for (int i = 0; i < keys.Length; i++) - { - if (dict.TryGetValue(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // SteadyState — the KEY L4 benchmark - // Pre-filled table at capacity, then 4096 operations: - // ~95% are lookups (cache hits) - // ~5% are miss → evict cold entry → insert new entry → lookup new - // This directly simulates L4's main runtime loop. - // ════════════════════════════════════════════════════════════ - - [Benchmark] - public nuint Dictionary_SteadyState() - { - nuint sum = 0; - Dictionary dict = _freshDict; - nuint[] resident = _residentKeys; - nuint[] external = _externalKeys; - int[] script = _steadyStateScript; - int[] evict = _evictOrder; - int evictCursor = 0; - int externalCursor = 0; - - for (int i = 0; i < script.Length; i++) - { - int op = script[i]; - if (op >= 0) - { - // Cache hit: lookup existing entry - if (dict.TryGetValue(resident[op], out nuint value)) - { - sum += value; - } - } else - { - // Cache miss: evict a cold entry, insert a new one, then lookup - int evictIdx = evict[evictCursor % evict.Length]; - evictCursor++; - dict.Remove(resident[evictIdx]); - - int extIdx = externalCursor % external.Length; - externalCursor++; - nuint newKey = external[extIdx]; - nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF); - dict[newKey] = newVal; - - if (dict.TryGetValue(newKey, out nuint v)) { - sum += v; - } - } - } - return sum; - } - - [IterationSetup(Targets = [ - nameof(SwissTable_Insert), - nameof(SwissTable_SteadyState), - nameof(Dictionary_Insert), - nameof(Dictionary_SteadyState) - ])] - public void IterationSetup() - { - _freshSwiss = SwissTable.Create(); - _freshDict = new Dictionary(KeyCount); - - // Pre-fill for steady-state benchmarks - for (int i = 0; i < KeyCount; i++) - { - nuint key = _residentKeys[i]; - nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); - _freshSwiss.TryAddOrSet(key, val, out _); - _freshDict[key] = val; - } - } - - // ════════════════════════════════════════════════════════════ - // Setup / Teardown - // ════════════════════════════════════════════════════════════ - - [GlobalSetup] - public void Setup() - { - Random rng = new Random(42); - - // ── Generate pointer-like keys (HeapBase + random_offset * 1024) ── - _residentKeys = GeneratePointerKeys(KeyCount, rng); - _externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation - - // ── Build lookup sequences ── - _zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng); - _uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng); - - // ── Build steady-state operation script ── - _steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng); - - // ── Build eviction order (cold tail entries first) ── - _evictOrder = new int[KeyCount]; - for (int i = 0; i < KeyCount; i++) { - _evictOrder[i] = i; - } - - // Shuffle so eviction isn't just sequential - Shuffle(_evictOrder, rng); - - // ── Pre-fill lookup tables ── - _swissLookup = SwissTable.Create(); - _dictLookup = new Dictionary(KeyCount); - - for (int i = 0; i < KeyCount; i++) - { - nuint key = _residentKeys[i]; - nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer - _swissLookup.TryAddOrSet(key, val, out _); - _dictLookup[key] = val; - } - } - - [Benchmark] - public void SwissTable_Insert() - { - SwissTable swiss = _freshSwiss; - nuint[] keys = _residentKeys; - for (int i = 0; i < keys.Length; i++) { - swiss.TryAddOrSet(keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF), out _); - } - } - - [Benchmark] - public nuint SwissTable_Lookup_Uniform() - { - nuint sum = 0; - nuint[] keys = _uniformLookups; - SwissTable swiss = _swissLookup; - for (int i = 0; i < keys.Length; i++) - { - if (swiss.TryGet(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - [Benchmark] - public nuint SwissTable_Lookup_Zipfian() - { - nuint sum = 0; - nuint[] keys = _zipfianLookups; - SwissTable swiss = _swissLookup; - for (int i = 0; i < keys.Length; i++) - { - if (swiss.TryGet(keys[i], out nuint value)) { - sum += value; - } - } - return sum; - } - - [Benchmark] - public nuint SwissTable_SteadyState() - { - nuint sum = 0; - SwissTable swiss = _freshSwiss; - nuint[] resident = _residentKeys; - nuint[] external = _externalKeys; - int[] script = _steadyStateScript; - int[] evict = _evictOrder; - int evictCursor = 0; - int externalCursor = 0; - - for (int i = 0; i < script.Length; i++) - { - int op = script[i]; - if (op >= 0) - { - // Cache hit: lookup existing entry - if (swiss.TryGet(resident[op], out nuint value)) - { - sum += value; - } - } else - { - // Cache miss: evict a cold entry, insert a new one, then lookup - int evictIdx = evict[evictCursor % evict.Length]; - evictCursor++; - swiss.TryRemove(resident[evictIdx], out _); - - int extIdx = externalCursor % external.Length; - externalCursor++; - nuint newKey = external[extIdx]; - nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF); - swiss.TryAddOrSet(newKey, newVal, out _); - - if (swiss.TryGet(newKey, out nuint v)) { - sum += v; - } - } - } - return sum; - } - - // ════════════════════════════════════════════════════════════ - // Data generation helpers - // ════════════════════════════════════════════════════════════ - - /// - /// Generate unique pointer-like keys. Each key = HeapBase + - /// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses - /// scattered across a ~1 GB virtual address range. - /// - private static nuint[] GeneratePointerKeys( - int count, - Random rng - ) - { - nuint[] keys = new nuint[count]; - HashSet used = new HashSet(count); - for (int i = 0; i < count; i++) - { - long offset; - do - { - // Random offset within ~1M cache lines (1 GB range) - offset = (long)rng.Next(0, 1 << 20) * CacheLineSize; - } while (!used.Add(offset)); - keys[i] = (nuint)(HeapBase + offset); - } - return keys; - } - - /// - /// Generate a steady-state operation script. Each entry: non-negative value = index into - /// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert → - /// lookup). Lookup indices are Zipfian-skewed. - /// - private static int[] GenerateSteadyStateScript( - int keyCount, - int opCount, - double missRate, - Random rng - ) - { - int hotSetSize = Math.Max(1, keyCount / 5); - int[] script = new int[opCount]; - for (int i = 0; i < opCount; i++) - { - if (rng.NextDouble() < missRate) - { - script[i] = -1; // miss - } else - { - // Zipfian hit - if (rng.NextDouble() < 0.8) - { - script[i] = rng.Next(0, hotSetSize); - } else - { - script[i] = rng.Next(hotSetSize, keyCount); - } - } - } - return script; - } - - /// - /// Generate uniform random lookup sequence (baseline for comparison with Zipfian). - /// - private static nuint[] GenerateUniformLookups( - nuint[] sourceKeys, - int count, - Random rng - ) - { - nuint[] lookups = new nuint[count]; - for (int i = 0; i < count; i++) { - lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)]; - } - return lookups; - } - - /// - /// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys - /// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This - /// models matrix computation where a small panel working set is reused heavily. - /// - private static nuint[] GenerateZipfianLookups( - nuint[] sourceKeys, - int count, - Random rng - ) - { - int hotSetSize = Math.Max(1, sourceKeys.Length / 5); - nuint[] lookups = new nuint[count]; - for (int i = 0; i < count; i++) - { - if (rng.NextDouble() < 0.8) - { - lookups[i] = sourceKeys[rng.Next(0, hotSetSize)]; - } else - { - lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)]; - } - } - return lookups; - } - - /// - /// Fisher-Yates shuffle. - /// - private static void Shuffle( - int[] array, - Random rng - ) - { - for (int i = array.Length - 1; i > 0; i--) - { - int j = rng.Next(0, i + 1); - (array[i], array[j]) = (array[j], array[i]); - } - } - - } -} diff --git a/src/Nitload.Common/Collections/SwissTable.X86.cs b/src/Nitload.Common/Collections/SwissTable.X86.cs deleted file mode 100644 index a2e01eac..00000000 --- a/src/Nitload.Common/Collections/SwissTable.X86.cs +++ /dev/null @@ -1,130 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="SwissTable.X86" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System; -using System.Numerics; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; - -namespace Nitload.Common.Collections -{ - public partial class SwissTable - { - - private unsafe class BucketScannerX86 : IBucketScanner - { - - public bool TryFind( - nuint key, - uint hash, - Bucket* bucket, - out nuint value - ) - { - byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); - Vector128 ctrlVector = Vector128.Create(ctrl); - Vector128 bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control); - Vector128 cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector); - int mask = Sse2.MoveMask(cmpResult); - while (mask != 0) - { - int bitPos = BitOperations.TrailingZeroCount(mask); - int index = bitPos; - if (bucket->Keys[index] == key) - { - value = bucket->Values[index]; - return true; - } - mask &= mask - 1; - } - value = default; - return false; - } - - public bool TryRemove( - nuint key, - uint hash, - Bucket* bucket, - out nuint value - ) - { - byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); - Vector128 ctrlVector = Vector128.Create(ctrl); - - Vector128 bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control); - Vector128 matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl); - int matches = Sse2.MoveMask(matchMask); - - while (matches != 0) - { - int bitPos = BitOperations.TrailingZeroCount(matches); - int index = bitPos; - if (bucket->Keys[index] == key) - { - value = bucket->Values[index]; - bucket->Control[index] = 0xFE; // mark as DELETED - // no need to remove Key/Value - return true; - } - matches &= matches - 1; - } - value = default; - return false; - } - - public bool TrySet( - nuint key, - uint hash, - Bucket* bucket, - nuint value, - out bool isNewKey - ) - { - byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0) - Vector128 ctrlVector = Vector128.Create(ctrl); - - Vector128 bucketCtrl = Sse2.LoadVector128( - (byte*)&bucket->Control); - - // === Phase 1: Look up an existing key === - Vector128 matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl); - int matches = Sse2.MoveMask(matchMask); - - while (matches != 0) - { - int bitPos = BitOperations.TrailingZeroCount(matches); - int index = bitPos; - if (bucket->Keys[index] == key) - { - // Key already exists -> update value - bucket->Values[index] = value; - isNewKey = false; - return true; - } - matches &= matches - 1; - } - - // === Phase 2: Track an available insertion slot === - int emptyOrDeleted = Sse2.MoveMask(bucketCtrl); - if (emptyOrDeleted == 0) - { - isNewKey = false; - return false; // bucket full - } - - int slot = BitOperations.TrailingZeroCount(emptyOrDeleted); - bucket->Control[slot] = ctrl; - bucket->Keys[slot] = key; - bucket->Values[slot] = value; - isNewKey = true; - return true; - } - - } - - } -} diff --git a/src/Nitload.Common/Collections/SwissTable.cs b/src/Nitload.Common/Collections/SwissTable.cs deleted file mode 100644 index 67bbc0a1..00000000 --- a/src/Nitload.Common/Collections/SwissTable.cs +++ /dev/null @@ -1,224 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="SwissTable" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -namespace Nitload.Common.Collections -{ - /// - /// - /// - public sealed unsafe partial class SwissTable : IDisposable - { - - private const int BucketCount = 1 << EntryMaskWidth; - private const int BucketSize = 16; - private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation - private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket - public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets - - public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf(); - - private readonly Entry* _entries; - private readonly Action _freeHandle; - private bool _disposed; - - private SwissTable( - Entry* buffer, - Action freeHandle - ) - { - _entries = buffer; - _freeHandle = freeHandle; - } - - public static SwissTable Create() - { - Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf(), 16); - for (int i = 0; i < BucketCount; i++) - { - // Set all control bytes to 0xFF (empty) - Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize); - } - return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle)); - } - - public static SwissTable Create( - Func allocator, - Action freeHandle - ) - { - Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize); - for (int i = 0; i < BucketCount; i++) - { - // Set all control bytes to 0xFF (empty) - Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize); - } - return new SwissTable(buffer, freeHandle); - } - - public void Dispose() - { - if (_disposed) { - return; - } - _disposed = true; - _freeHandle((nuint)_entries); - GC.SuppressFinalize(this); - } - - public bool TryAddOrSet( - nuint key, - nuint value, - out bool isNewKey - ) - { - uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key); - ushort entry = (ushort)(hash & (BucketCount - 1)); - Bucket* bucket = &(*_entries)[entry]; - return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey); - } - - public bool TryGet( - nuint key, - out nuint value - ) - { - uint hash = BitOperations.Crc32C(CrcMagic, key); - ushort entry = (ushort)(hash & (BucketCount - 1)); - Bucket* bucket = &(*_entries)[entry]; - return IBucketScanner.Current.TryFind(key, hash, bucket, out value); - /* - * hash = CRC32 of key - * entry = low 8 of hash - * ctrl = 8-15 of hash - * index = rest of hash - * find value of hash - */ - } - - public bool TryRemove( - nuint key, - out nuint value - ) - { - uint hash = BitOperations.Crc32C(CrcMagic, key); - ushort entry = (ushort)(hash & (BucketCount - 1)); - Bucket* bucket = &(*_entries)[entry]; - return IBucketScanner.Current.TryRemove(key, hash, bucket, out value); - } - - internal static SwissTable Create( - nuint bufferPtr, - Action FreeHandle - ) - { - Entry* buffer = (Entry*)bufferPtr; - for (int i = 0; i < BucketCount; i++) - { - // Set all control bytes to 0xFF (empty) - Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize); - } - return new SwissTable(buffer, FreeHandle); - } - - ~SwissTable() - { - Dispose(); - } - - [InlineArray(BucketCount)] - private struct Entry - { - - public Bucket Bucket; - - } - - private unsafe struct Bucket - { - - public BucketControl Control; - public BucketKeyStorage Keys; - public BucketValueStorage Values; - - } - - private interface IBucketScanner - { - - static IBucketScanner Current { get; } = Create(); - - bool TryFind( - nuint key, - uint Hash, - Bucket* bucket, - out nuint value - ); - - unsafe bool TryRemove( - nuint key, - uint hash, - Bucket* bucket, - out nuint value - ); - - bool TrySet( - nuint key, - uint hash, - Bucket* bucket, - nuint value, - out bool isNewKey - ); - - private static IBucketScanner Create() - { - if (Sse42.IsSupported) - { - return new BucketScannerX86(); - } else if (ArmBase.IsSupported) - { - throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform."); - } else - { - throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform."); - } - } - - } - - [InlineArray(BucketSize)] - private struct BucketControl - { - - public byte Control; - - } - - [InlineArray(BucketSize)] - private struct BucketKeyStorage - { - - public nuint value; - - } - - [InlineArray(BucketSize)] - private struct BucketValueStorage - { - - public nuint value; - - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs index 575bd318..7c605e60 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -5,7 +5,6 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.Common.HighPerformance.ProcessorAbstraction; using System.Runtime.InteropServices; namespace Nitload.Common.HighPerformance.Memory @@ -23,15 +22,11 @@ namespace Nitload.Common.HighPerformance.Memory } [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)] - private unsafe struct CacheLineMetadata + private struct CacheLineMetadata { internal const uint MaxBufferSize = 0xFFFFFFFF - 64U; - [FieldOffset(0)] private readonly M256 FillingMask; - - [FieldOffset(24)] internal nuint Origin; - public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); #region tlsf linked list @@ -50,32 +45,17 @@ namespace Nitload.Common.HighPerformance.Memory #endregion - #region mask - [FieldOffset(16)] internal AllocationState State; - [FieldOffset(17)] internal byte ReversedProfile; - [FieldOffset(18)] internal ushort CustomizeProfile; - [FieldOffset(20)] internal int RefCount; - #endregion } #pragma warning disable IDE0032 - [StructLayout(LayoutKind.Explicit)] - public unsafe struct Handle : IDisposable + [StructLayout(LayoutKind.Explicit, Size = 16)] + public struct Handle : IDisposable { - [FieldOffset(8)] private readonly CacheLineMetadata* _head; [FieldOffset(0)] private nuint _handle; - - private Handle( - nuint handle, - CacheLineMetadata* cacheRefCount - ) - { - _handle = handle; - _head = cacheRefCount; - } + [FieldOffset(8)] private nuint _reserved; [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] public Handle() { } @@ -88,7 +68,7 @@ namespace Nitload.Common.HighPerformance.Memory return; } _handle = 0; - _ = Interlocked.Decrement(ref _head->RefCount); + _reserved = 0; } } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs deleted file mode 100644 index 53a40633..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs +++ /dev/null @@ -1,92 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Debug" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - /**/ - public partial class L4 - { - - public void DebugAgeAll() - { - // AgeAll(); - } - - public bool DebugContains( - nuint origin - ) - { - return _map.TryGet(origin, out _); - } - - public int DebugEvict( - int count = 1 - ) - { - return Evict(count); - } - - public unsafe DebugSnapshot GetDebugSnapshot() - { - throw new NotImplementedException(); - /* - int maxId = _maxId; - byte[] rrpv = new byte[maxId]; - byte* p = (byte*)_eviction; - for (int i = 0; i < maxId; i++) { - rrpv[i] = p[i]; - } - return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv); - /**/ - } - - public readonly struct DebugSnapshot - { - - /// - /// RRPV value for each line（0~3）, length == MaxId - /// - public readonly byte[] RrpvSnapshot; - /// - /// Count of activated cache line handle - /// - public readonly int ActiveCount; - - /// - /// Count of predictor ahead of current map position（aheadCount） - /// - public readonly int AheadCount; - - /// - /// Max ID of the line being using or once used（maxId） - /// - public readonly int MaxId; - - internal DebugSnapshot( - int activeCount, - int aheadCount, - int maxId, - byte[] rrpv - ) - { - ActiveCount = activeCount; - AheadCount = aheadCount; - MaxId = maxId; - RrpvSnapshot = rrpv; - } - - } - - } - /**/ -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs deleted file mode 100644 index f53be977..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs +++ /dev/null @@ -1,64 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Evict" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -namespace Nitload.Common.HighPerformance.Memory -{ - public partial class L4 - { - - /// - /// - /// - /// - /// - /// - /// - /// Count of lines not evicted - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe int Evict( - int count = 1 - ) - { - count = (count < 0) ? 0 : count; - int* pPtr = _predictCount, mPtr = _mapCount; - - - int remain = count; - for (int i = 0; i < CacheLineCount; i++) - { - if (pPtr[i] > mPtr[i]) - { - continue; - } - remain -= TryEvictAt(i) ? 1 : 0; - } - return remain; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe bool TryEvictAt( - int index - ) - { - CacheLineMetadata* head = &_head[index]; - if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0) - { - // EvictLine(i + index); - _ = _map.TryRemove(head->Origin, out _); - IL4Predict predict = CurrentPredict; - predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask)); - _bufferAllocator.Free(index); - _mapCount[index] = 0; - _predictCount[index] = int.MaxValue; - return true; - } - return false; - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs deleted file mode 100644 index de4efadb..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Exceptions" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -namespace Nitload.Common.HighPerformance.Memory -{ -#pragma warning disable CA1032 // Implement standard exception constructors - - public sealed class L4PredictUnregisteredException : Exception - { - - public L4PredictUnregisteredException( - int id - ) - : base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { } - - } - - public sealed class L4PredictConflictException : Exception - { - - public L4PredictConflictException( - nuint basePtr, - ushort policy, - int current - ) - : base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { } - - } -#pragma warning restore CA1032 // Implement standard exception constructors -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs deleted file mode 100644 index d30b411c..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs +++ /dev/null @@ -1,38 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Predict" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using Nitload.Common.HighPerformance.Memory; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - public partial class L4 - { - - private IL4Predict CurrentPredict { get; set; } - - public void ResetPredict( - IL4Predict predict - ) - { - if (CurrentPredict is null) - { - CurrentPredict = predict; - return; - } - - // TODO: write back all and register new predictor - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs deleted file mode 100644 index 4344483c..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs +++ /dev/null @@ -1,146 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Predictor" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload.Common.HighPerformance.Memory -{ - [Flags] - public enum PredictResult - { - - Success = 1, - Exhausted = 2, - - } - - public interface IL4Predict - { - - PredictResult Predict( - Span node, - out int count - ); - - /// - /// Materializes the cache line identified by into . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Destination cache line buffer. - /// - void Prefetch( - nuint origin, - uint policy, - Span cache - ); - - /// - /// Writes the cache line in back to . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Source cache line buffer. - /// - void WriteBack( - nuint origin, - uint policy, - ReadOnlySpan cache - ); - - } - - [StructLayout(LayoutKind.Explicit)] - public struct CacheLineDescription : IEquatable - { - - public override readonly bool Equals( - object? obj - ) - { - return (obj is CacheLineDescription prediction) && - Address.Equals(prediction.Address) && - (MapPolicy == prediction.MapPolicy); - } - - public readonly bool Equals( - CacheLineDescription other - ) - { - return - (Address == other.Address) && (MapPolicy == other.MapPolicy); - } - - public override readonly int GetHashCode() - { - return (int)ComposePredictionKey64(Address, MapPolicy); - } - - public readonly nuint GetHashCodeLong() - { - return ComposePredictionKey64(Address, MapPolicy); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static nuint ComposePredictionKey64( - nuint address, - ushort mapPolicy - ) - { - nuint x = address; - nuint y = mapPolicy; - unchecked - { - nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL)); - mixed ^= mixed >> 30; - mixed *= (nuint)0xbf58476d1ce4e5b9UL; - mixed ^= mixed >> 27; - mixed *= (nuint)0x94d049bb133111ebUL; - mixed ^= mixed >> 31; - return mixed; - } - } - - public static bool operator !=( - CacheLineDescription left, - CacheLineDescription right - ) - { - return !(left == right); - } - public static bool operator ==( - CacheLineDescription left, - CacheLineDescription right - ) - { - return left.Equals(right); - } - -#pragma warning disable CA1051 - [FieldOffset(0)]public nuint Address; - [FieldOffset(8)]public uint Size; - [FieldOffset(12)]public ushort MapPolicy; - [FieldOffset(14)]internal short Predictor; -#pragma warning restore CA1051 - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs deleted file mode 100644 index 73c7694e..00000000 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs +++ /dev/null @@ -1,102 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="L4.Prefetch" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using System.ComponentModel; -using System.Numerics; - -namespace Nitload.Common.HighPerformance.Memory -{ - public unsafe partial class L4 - { - - private partial void Prefetch() - { - if (CurrentPredict is null) { - return; - } - IL4Predict predict = CurrentPredict; - Span span = new(_prediction, PredictionCount); - PredictResult result = predict.Predict(span, out int actualCount); - _exhausted = (result & PredictResult.Exhausted) != 0; - if (actualCount <= 0) - { - _exhausted = true; - return; - } - int missIndex = 0; - for (int i = 0; i < actualCount; i++) - { - ref CacheLineDescription handle = ref span[i]; - nuint address = handle.Address; - if (!_map.TryGet(address, out _)) - { - // miss and record - span[missIndex] = handle; - missIndex++; - } - } - - // now prefetch the misses - for (int i = 0; i < missIndex; i++) - { - CacheLineMetadata* head; - ref CacheLineDescription pHandle = ref span[i]; - nuint address = pHandle.Address; - int index; - if (_map.TryGet(address, out nuint handle)) - { - head = (CacheLineMetadata*)handle; - index = (int)(head - _head); - _predictCount[index] = 0; // reset eviction counter - - #region update predictor belonging - - if (_predictCount[index] > short.MaxValue / 2) - { - int predictCount = Volatile.Read(ref _predictCount[index]); - int mapCount = Volatile.Read(ref _mapCount[index]); - int decrement = (int.Min(predictCount, mapCount) + 1) / 2; - _ = Interlocked.Add(ref _predictCount[index], -decrement); - _ = Interlocked.Add(ref _mapCount[index], -decrement); - } - _predictCount[index]++; - - #endregion - - continue; - } - - int retryCount = 0, remain = missIndex - i; - const int maxRetry = 1024; - while (!_bufferAllocator.TryAlloc(pHandle.Size, out index)) - { - remain = Evict(remain); - retryCount++; - if (retryCount > maxRetry) { - throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); - } - if (retryCount > maxRetry / 2) { - Thread.Sleep(1); - } - } - head = _head + index; - head->Origin = address; - head->CustomizeProfile = pHandle.MapPolicy; - head->RefCount = 0; - - _predictCount[index] = 1; - _mapCount[index] = 1; - - // pre fetch them here - predict.Prefetch(head->Origin, head->CustomizeProfile, new Span((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask)); - _ = _map.TryAddOrSet(address, (nuint)head, out _); - } - _ = Interlocked.Add(ref _aheadCount, actualCount); - } - - } -} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs new file mode 100644 index 00000000..f87d40bd --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs @@ -0,0 +1,217 @@ +// ----------------------------------------------------------------------------- +// file="L4.SwissTable" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Nitload.Common.HighPerformance.Memory +{ + public sealed unsafe partial class L4 + { + + private const int BucketCapacity = 16; + private const uint CrcMagic = 0x7f1c9961; + + private const int GroupCount = 1 << GroupCountLog; + private const int GroupCountLog = 10; + private const int IdealCount = GroupCount * BucketCapacity;//16384 + private const int OffsetOfBucket = IdealCount; + + private static readonly int TableBufferSize = + IdealCount + (GroupCount * Unsafe.SizeOf()); + internal static readonly int BufferSize = TableBufferSize * 2; + + private enum SwissControl : byte + { + + Empty = 0b10000000, + Deleted = 0b10000001 + + } + + [StructLayout(LayoutKind.Sequential)] + internal struct AddressSource + { + + public ulong Address; + public ulong Description; + + public override readonly bool Equals( + object obj + ) + { + return obj is AddressSource source && this == source; + } + + public override readonly int GetHashCode() + { + return HashCode.Combine(Address, Description); + } + + public static bool operator !=( + AddressSource left, + AddressSource right + ) + { + return !(left == right); + } + + public static bool operator ==( + AddressSource left, + AddressSource right + ) + { + return left.Address == right.Address && left.Description == right.Description; + } + + } + + [InlineArray(BucketCapacity)] + internal struct KeyArray + { + + private AddressSource _element0; + + } + + internal struct SwissBucket + { + + public KeyArray Keys; + public fixed ulong Values[BucketCapacity]; + + } + + internal sealed unsafe class SwissTable : IDisposable + { + + private byte* _buffer_read; + private byte* _buffer_write; + private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion); + + private SwissTable(byte* buffers) + { + _buffer_read = buffers; + _buffer_write = buffers + TableBufferSize; + InitializeBuffer(_buffer_read); + InitializeBuffer(_buffer_write); + } + + internal static SwissTable Create(byte* buffers) + { + ArgumentNullException.ThrowIfNull(buffers); + return new SwissTable(buffers); + } + + /// + /// Reads the published buffer. Callers must prevent a swap while the read is in progress. + /// + public bool TryGet(AddressSource key, out ulong value) + { + _synchronization.EnterReadLock(); + try + { + return SwissReaderX86.TryGet(_buffer_read, key, out value); + } + finally + { + _synchronization.ExitReadLock(); + } + } + + /// + /// Writes to the inactive buffer. Calls must be serialized by the caller. + /// + public bool TryAddOrSet(AddressSource key, ulong value) + { + _synchronization.EnterWriteLock(); + try + { + return SwissWriterX86.TryAddOrSet(_buffer_write, key, value); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Writes to the inactive buffer. Calls must be serialized by the caller. + /// + public bool TryRemove(AddressSource key, out ulong value) + { + _synchronization.EnterWriteLock(); + try + { + return SwissWriterX86.TryRemove(_buffer_write, key, out value); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced. + /// + public void PrepareWriteBuffer() + { + _synchronization.EnterWriteLock(); + try + { + InitializeBuffer(_buffer_write); + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + /// + /// Publishes the write buffer. The caller must provide the phase barrier around this operation. + /// + public void SwapBuffers() + { + _synchronization.EnterWriteLock(); + try + { + byte* temporary = _buffer_read; + _buffer_read = _buffer_write; + _buffer_write = temporary; + } + finally + { + _synchronization.ExitWriteLock(); + } + } + + public void Dispose() + { + _synchronization.EnterWriteLock(); + try + { + _buffer_read = null; + _buffer_write = null; + } + finally + { + _synchronization.ExitWriteLock(); + } + _synchronization.Dispose(); + } + + private static void InitializeBuffer(byte* buffer) + { + Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount); + } + + } + + } +} \ No newline at end of file diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs new file mode 100644 index 00000000..a19f7944 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs @@ -0,0 +1,200 @@ +// ----------------------------------------------------------------------------- +// file="L4.Swiss_X86" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System.Numerics; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace Nitload.Common.HighPerformance.Memory +{ + public partial class L4 + { + + private readonly unsafe struct SwissWriterX86 + { + + public static bool TryAddOrSet( + byte* buffer, + AddressSource key, + ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + + uint first_tomb_group = 0; + int first_tomb_idx = -1; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128* control_bytes_ptr = control_bytes + begin; + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr); + + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted); + + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + Vector128 cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128); + + int hash_mask = Sse2.MoveMask(cmp_hash); + int empty_mask = Sse2.MoveMask(cmp_empty); + int tomb_mask = Sse2.MoveMask(cmp_tomb); + + if (tomb_mask != 0 && first_tomb_idx < 0) + { + int index = BitOperations.TrailingZeroCount(tomb_mask); + first_tomb_group = begin; + first_tomb_idx = index; + } + while (hash_mask != 0) + { + int index = BitOperations.TrailingZeroCount(hash_mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + bucket->Values[index] = value; + return true; + } + hash_mask &= ~(1 << index); + } + while (empty_mask != 0) + { + int index = BitOperations.TrailingZeroCount(empty_mask); + if (first_tomb_idx < 0) + { + SwissBucket* bucket = &buckets[begin]; + bucket->Keys[index] = key; + bucket->Values[index] = value; + ((byte*)control_bytes_ptr)[index] = target_hash; + } else + { + SwissBucket* bucket = &buckets[first_tomb_group]; + bucket->Keys[first_tomb_idx] = key; + bucket->Values[first_tomb_idx] = value; + ((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash; + } + return true; + } + + begin = (begin + 1) & (GroupCount - 1); + } + if (first_tomb_idx >= 0) + { + SwissBucket* bucket = &buckets[first_tomb_group]; + bucket->Keys[first_tomb_idx] = key; + bucket->Values[first_tomb_idx] = value; + ((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash; + return true; + } + value = 0; + return false; + } + + public static bool TryRemove( + byte* buffer, + AddressSource key, + out ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin)); + + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + + int mask = Sse2.MoveMask(cmp_hash); + + while (mask != 0) + { + int index = BitOperations.TrailingZeroCount(mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + ((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted; + value = bucket->Values[index]; + return true; + } + mask &= ~(1 << index); + } + + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + int empty_mask = Sse2.MoveMask(cmp_empty); + if (empty_mask != 0) + { + break; + } + begin = (begin + 1) & (GroupCount - 1); + } + value = 0; + return false; + } + + } + + private readonly unsafe struct SwissReaderX86 + { + + public static bool TryGet( + byte* buffer, + AddressSource key, + out ulong value + ) + { + Vector128* control_bytes = (Vector128*)buffer; + SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); + + uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint begin = hash % GroupCount; + byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); + + for (int i = 0; i < GroupCount; i++) + { + Vector128 control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin)); + Vector128 target_hash_v128 = Vector128.Create(target_hash); + Vector128 cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128); + int mask = Sse2.MoveMask(cmp_hash); + while (mask != 0) + { + int index = BitOperations.TrailingZeroCount(mask); + SwissBucket* bucket = &buckets[begin]; + if (bucket->Keys[index] == key) + { + value = bucket->Values[index]; + return true; + } + mask &= ~(1 << index); + } + + Vector128 target_empty_v128 = Vector128.Create((byte)SwissControl.Empty); + Vector128 cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128); + if (Sse2.MoveMask(cmp_empty) != 0) + break; + + begin = (begin + 1) & (GroupCount - 1); + } + value = 0; + return false; + } + + } + + } +} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs index 1b9a873a..40a8a396 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs @@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory public partial class L4 { - private unsafe class TLSF + private unsafe struct TLSF { public const int MaxCacheLineCount = 8192; - public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf; - - private readonly short[] _bucket; + public static readonly nuint RequestedAllocation = + ((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf; private readonly CacheLineMetadata* _spareMemory; private readonly int _align; private readonly int _maxLevel; private int _metadataEnd; private readonly Ptr64 _basePtr; + + private fixed short _bucket[64]; private readonly uint _maxAllocSize; private readonly uint _totalSize; @@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory _maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U)); _maxLevel = int.MaxValue; _maxLevel = GetLevel(_maxAllocSize); - _bucket = new short[GetLevel(_maxAllocSize) + 2]; + + // _bucket = new short[GetLevel(_maxAllocSize) + 2]; + _bucket[63] = (short)(GetLevel(_maxAllocSize) + 2); _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _metadataEnd = 0; @@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory idx = candidate->NextLevel; } } - for (int i = level + 1; i < _bucket.Length; i++) + for (int i = level + 1; i < _bucket[63]; i++) { int idx = _bucket[i]; if (idx != 0) diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.cs b/src/Nitload.Common/HighPerformance/Memory/L4.cs index cd7e7a48..e0c30f3f 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.cs @@ -5,49 +5,20 @@ // SPDX-License-Identifier: MIT // ----------------------------------------------------------------------------- -using Nitload.Common.Collections; -using Nitload.Common.HighPerformance.Memory; using Nitload.Common.HighPerformance.ProcessorAbstraction; -using System.Data; -using System.Runtime.InteropServices; namespace Nitload.Common.HighPerformance.Memory { public sealed unsafe partial class L4 : IDisposable { - private const int CacheLineCount = 8192; private const nuint PageSize = 4096; - private const int PredictionCount = 2048; - private const int PredictionLeastAhead = 1024; - /// - /// Array of metadata for each cache line, also used as metadata for tlsf allocator. - /// - private readonly CacheLineMetadata* _head; - /// - /// Count of cache lines mapped for each prediction, used for eviction policy. - /// - private readonly int* _mapCount; - /// - /// Count of times each cache line is predicted, used for eviction policy. - /// - private readonly int* _predictCount; - /// - /// Original memory view for prediction span. - /// - private readonly CacheLineDescription* _prediction; private bool _disposed; - private bool _exhausted; - - private int _aheadCount; - private readonly int _bindedNuma; - private volatile int _prefetching; private readonly nuint _baseAddress; - private readonly SwissTable _map; - - private readonly TLSF _bufferAllocator; + private SwissTable _addressMap; + private TLSF _bufferAllocator; public L4( int bindedNuma, @@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize); ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize); _bindedNuma = bindedNuma; - - // init order is fallow by align requirement of each component. - nuint headSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align - nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align - nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align - nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align - nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size + nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); ByteCapacity = normalizedBufferSize; - - AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize; - - // malloc all memory at once, and free at once in Dispose. - // This is to avoid fragmentation and improve performance. + nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation; + AllocSize = swissBufferOffset + (nuint)BufferSize; _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize); - nuint ptr = _baseAddress; - - // init all components with the buffer, and move the buffer pointer forward. - nuint _buffer = _baseAddress; _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress); - _buffer += normalizedBufferSize; - _head = (CacheLineMetadata*)ptr; - ptr += headSize; - _prediction = (CacheLineDescription*)ptr; - ptr += predictionSize; - _map = SwissTable.Create(ptr, _ => { }); - ptr += mapCountSize; - _predictCount = (int*)ptr; - ptr += predictionCountSize; - _mapCount = (int*)ptr; - ptr += predictionCountSize; - - M512* mPtr = (M512*)_head; - for (uint i = 1; i < CacheLineCount; i++) { - mPtr->BroadCastFrom(0); - } + _addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset)); } public nuint ByteCapacity { get; init; } public nuint AllocSize { get; init; } - // public IL4Predict PrefetchPredict { get; set; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ContinueOnce( - Handle line - ) - { - if (line.Address > 0) { - _ = Interlocked.Decrement(ref _aheadCount); - } - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Design", - "CA1063:Implement IDisposable correctly", - Justification = """ - Only owns native memory via Numa.Free(); - no managed IDisposable fields. - Dispose(bool) pattern is unnecessary. - """)] public void Dispose() { if (_disposed) { return; } _disposed = true; - _map.Dispose(); -#pragma warning disable CA1816 - GC.SuppressFinalize(_bufferAllocator); -#pragma warning restore CA1816 - - - // init order is fallow by align requirement of each component. + _addressMap.Dispose(); Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize); GC.SuppressFinalize(this); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Handle Map( - ref CacheLineDescription prediction - ) + internal void Collect() { - if ((_aheadCount < PredictionLeastAhead) && !_exhausted) - { - if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) - { - try - { - if (_aheadCount < PredictionLeastAhead) - { - // do prefetch in work load now, sync - Prefetch(); - } - } - finally - { - _prefetching = 0; - } - } - } - while (true) - { - if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle)) - { - // cache hit - CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle; - if (Interlocked.Increment(ref entry->RefCount) > 0) - { - _predictCount[entry - _head] = 0; - _ = Interlocked.Decrement(ref _aheadCount); - _ = Interlocked.Decrement(ref _mapCount[entry - _head]); - nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry }; - return *(Handle*)hPtr; - } else - { - _ = Interlocked.Decrement(ref entry->RefCount); - } - } - Thread.Sleep(0); - } + ObjectDisposedException.ThrowIf(_disposed, this); + _bufferAllocator.Collect(); } - private partial void Prefetch(); + internal void Free( + int handle + ) + { + ObjectDisposedException.ThrowIf(_disposed, this); + _bufferAllocator.Free(handle); + } + + internal bool TryAllocate( + uint size, + out int handle + ) + { + ObjectDisposedException.ThrowIf(_disposed, this); + return _bufferAllocator.TryAlloc(size, out handle); + } } } diff --git a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs index b4ada2af..a15550cf 100644 --- a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs +++ b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs @@ -16,22 +16,21 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction public readonly struct LogicalCore { - public readonly uint GlobalCpuId; - public readonly uint NumaGroupId; - - /// - /// Stays 0 on linux. - /// + public readonly uint ProcessorGroupId; + public readonly uint SocketId; + public readonly uint CpuIndex; public readonly uint NumaNodeId; internal LogicalCore( - uint globalCpuId, - uint numaGroupId, + uint processorGroupId, + uint socketId, + uint cpuIndex, uint numaNodeId ) { - GlobalCpuId = globalCpuId; - NumaGroupId = numaGroupId; + ProcessorGroupId = processorGroupId; + SocketId = socketId; + CpuIndex = cpuIndex; NumaNodeId = numaNodeId; } diff --git a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs index d2100eb1..beb0f333 100644 --- a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs +++ b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs @@ -33,7 +33,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction if (_numaNodes.Length == 0) { throw new InvalidOperationException("Numa info has not been loaded."); } - if ((numaId < 0) || (numaId >= _numaNodes.Length)) { + if ((numaId < 0) || !ContainsNode((uint)numaId)) { throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID."); } return NumaAllocNative(size, numaId); @@ -69,7 +69,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction if (_numaNodes.Length == 0) { throw new InvalidOperationException("Numa info has not been loaded."); } - if ((numaId < 0) || (numaId >= _numaNodes.Length)) { + if ((numaId < 0) || !ContainsNode((uint)numaId)) { throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID."); } NumaFreeNative(address, size, numaId); @@ -80,8 +80,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] public static partial int GetNumaEach( uint numaId, - Span CpuIndexInNuma, - Span globalCpuIdInNuma + Span processorGroupIds, + Span socketIds, + Span cpuIndices, + Span numaNodeIds ); [LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_overall")] @@ -97,10 +99,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction if (_numaNodes.Length == 0) { throw new InvalidOperationException("Numa info has not been loaded."); } - L4[] buffers = new L4[NumaNodeCount]; - for (int i = 0; i < NumaNodeCount; i++) + L4[] buffers = new L4[_numaNodes.Length]; + for (int i = 0; i < _numaNodes.Length; i++) { - L4 buffer = new L4(i); + L4 buffer = new L4((int)_numaNodes[i].NumaNodeId); buffers[i] = buffer; } return buffers; @@ -108,14 +110,29 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction internal static void Initialize() { - Span coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1]; int numaNodeCount = 0; - if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) { + if (0 == GetNumaOverall(ref numaNodeCount, default)) { throw new InvalidOperationException("Failed to get NUMA overall information."); } - _numaNodes = new NumaNode[numaNodeCount]; - for (uint i = 0; i < numaNodeCount; i++) { - _numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]); + int[] coreCountInNuma = new int[numaNodeCount]; + if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) { + throw new InvalidOperationException("Failed to get NUMA node processor counts."); + } + int activeNodeCount = 0; + for (int i = 0; i < coreCountInNuma.Length; i++) { + if (coreCountInNuma[i] > 0) { + activeNodeCount++; + } + } + NumaNodeCount = activeNodeCount; + _numaNodes = new NumaNode[activeNodeCount]; + int nodeIndex = 0; + for (uint numaId = 0; numaId < numaNodeCount; numaId++) { + int coreCount = coreCountInNuma[(int)numaId]; + if (coreCount == 0) { + continue; + } + _numaNodes[nodeIndex++] = InitNumaNode(numaId, coreCount); } } @@ -123,8 +140,8 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] internal static partial void SetThreadAffinity( - int coreId, - int numaId + int cpuIndex, + int processorGroupId ); [MethodImpl(MethodImplOptions.NoInlining)] @@ -133,9 +150,24 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction int count ) { - Span globalIndex = stackalloc uint[count]; - _ = GetNumaEach(numaId, null, globalIndex); - return new(numaId, count, globalIndex); + Span processorGroupIds = stackalloc uint[count]; + Span socketIds = stackalloc uint[count]; + Span cpuIndices = stackalloc uint[count]; + Span numaNodeIds = stackalloc uint[count]; + if (0 == GetNumaEach(numaId, processorGroupIds, socketIds, cpuIndices, numaNodeIds)) { + throw new InvalidOperationException($"Failed to enumerate processors for NUMA node {numaId}."); + } + return new(numaId, count, processorGroupIds, socketIds, cpuIndices, numaNodeIds); + } + + private static bool ContainsNode(uint numaId) + { + foreach (NumaNode node in _numaNodes) { + if (node.NumaNodeId == numaId) { + return true; + } + } + return false; } [LibraryImport(Native.LibName, EntryPoint = "nfn_numa_alloc")] diff --git a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs index 94d75b4b..af4971ee 100644 --- a/src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs +++ b/src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs @@ -22,16 +22,26 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction internal NumaNode( uint numaId, int count, - Span logicalCoreIds + Span processorGroupIds, + Span socketIds, + Span cpuIndices, + Span numaNodeIds ) { + NumaNodeId = numaId; _logicalCores = new LogicalCore[count]; for (int i = 0; i < count; i++) { - _logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]); + _logicalCores[i] = new LogicalCore( + processorGroupIds[i], + socketIds[i], + cpuIndices[i], + numaNodeIds[i]); } } + public uint NumaNodeId { get; } + public int LogicalCoreCount => _logicalCores.Length; public ReadOnlySpan LogicalCores => _logicalCores; diff --git a/src/Nitload.Common/Threading/ThreadHelper.cs b/src/Nitload.Common/Threading/ThreadHelper.cs index 1e7f20af..9cf4cd8d 100644 --- a/src/Nitload.Common/Threading/ThreadHelper.cs +++ b/src/Nitload.Common/Threading/ThreadHelper.cs @@ -37,7 +37,7 @@ namespace Nitload.Threading LogicalCore core ) { - SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId); + SetThreadAffinity(core.CpuIndex, core.ProcessorGroupId); } [Conditional("DEBUG")] @@ -55,8 +55,8 @@ namespace Nitload.Threading [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] private static partial void SetThreadAffinity( - uint globalCoreId, - uint numaNodeId + uint cpuIndex, + uint processorGroupId ); #pragma warning restore CA5393 diff --git a/src/Nitload.Common/Timing/StgSharpTime.cs b/src/Nitload.Common/Timing/StgSharpTime.cs index f4c8422e..d8451b65 100644 --- a/src/Nitload.Common/Timing/StgSharpTime.cs +++ b/src/Nitload.Common/Timing/StgSharpTime.cs @@ -97,7 +97,7 @@ namespace Nitload } Thread.Sleep(10); - Numa.SetThreadAffinity((int)threadConfig.BindedCoreIndex, (int)threadConfig.BindedNumaNode); + Numa.SetThreadAffinity((int)threadConfig.CpuIndex, (int)threadConfig.ProcessorGroupId); while (mainProvider.IsRunning) { long elapsedTicks; diff --git a/src/Nitload.Common/Timing/TimeSourceProvider.cs b/src/Nitload.Common/Timing/TimeSourceProvider.cs index 73bd5776..395c26ba 100644 --- a/src/Nitload.Common/Timing/TimeSourceProvider.cs +++ b/src/Nitload.Common/Timing/TimeSourceProvider.cs @@ -14,15 +14,15 @@ using System.Threading; namespace Nitload.Common.Timing { public class TimeSourceStartupConfiguration( - nuint bindedNumaNode = 0, - nuint bindedCoreIndex = 0, + nuint processorGroupId = 0, + nuint cpuIndex = 0, ThreadPriority threadPriority = ThreadPriority.Highest ) { - public nuint BindedNumaNode { get; } = bindedNumaNode; + public nuint ProcessorGroupId { get; } = processorGroupId; - public nuint BindedCoreIndex { get; } = bindedCoreIndex; + public nuint CpuIndex { get; } = cpuIndex; public ThreadPriority Priority { get; } = threadPriority; diff --git a/src/Nitload.Native/.clangd b/src/Nitload.Native/.clangd index d7623dbb..29788b89 100644 --- a/src/Nitload.Native/.clangd +++ b/src/Nitload.Native/.clangd @@ -1,2 +1,2 @@ CompileFlags: - CompilationDatabase: ../.. + CompilationDatabase: . diff --git a/src/Nitload.Native/README.md b/src/Nitload.Native/README.md index 01e3d6b1..bf506d74 100644 --- a/src/Nitload.Native/README.md +++ b/src/Nitload.Native/README.md @@ -53,11 +53,11 @@ does not cross-compile the C library. ## clangd in VS Code -The `.clangd` file searches for `compile_commands.json` at the repository root. +The `.clangd` file searches for `compile_commands.json` in `src/Nitload.Native`. When using CMake Tools, set `cmake.copyCompileCommands` to -`${workspaceFolder}/compile_commands.json` so the active Debug or Release preset +`${workspaceFolder}/src/Nitload.Native/compile_commands.json` so the active Debug or Release preset updates that file after configuration. For command-line configuration, copy -`cmake_build//compile_commands.json` to the repository root. Restart +`cmake_build//compile_commands.json` to `src/Nitload.Native`. Restart clangd after changing the database or renaming headers. ## Code Layout diff --git a/src/Nitload.Native/nfn-src/thread.linux.c b/src/Nitload.Native/nfn-src/thread.linux.c index 324fbc77..8adbdd3c 100644 --- a/src/Nitload.Native/nfn-src/thread.linux.c +++ b/src/Nitload.Native/nfn-src/thread.linux.c @@ -26,9 +26,18 @@ typedef struct nfn_linux_max_context { int max_value; } nfn_linux_max_context; +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c typedef struct nfn_linux_fill_cpu_context { uint64_t *internal; uint32_t *global; +======= +typedef struct nif_linux_fill_cpu_context { + uint32_t *processor_group_ids; + uint32_t *socket_ids; + uint32_t *cpu_indices; + uint32_t *numa_node_ids; + int numa_id; +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c int index; } nfn_linux_fill_cpu_context; @@ -118,14 +127,43 @@ static int nfn_linux_track_max_id(int value, void *context) return 1; } +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c static int nfn_linux_fill_cpu_id(int value, void *context) { nfn_linux_fill_cpu_context *fill_context = (nfn_linux_fill_cpu_context *)context; +======= +static int nif_linux_read_socket_id(int cpu_id, uint32_t *socket_id) +{ + char path[160]; + char buffer[64]; + char *end = NULL; + long parsed_value; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu_id); + if (!nif_linux_read_first_line(path, buffer, sizeof(buffer))) + return 0; - if (fill_context->internal != NULL) - fill_context->internal[fill_context->index] = (uint64_t)fill_context->index; - if (fill_context->global != NULL) - fill_context->global[fill_context->index] = (uint32_t)value; + errno = 0; + parsed_value = strtol(buffer, &end, 10); + if (end == buffer || errno == ERANGE || parsed_value < 0 || parsed_value > UINT32_MAX) + return 0; + + *socket_id = (uint32_t)parsed_value; + return 1; +} + +static int nif_linux_fill_cpu_id(int value, void *context) +{ + nif_linux_fill_cpu_context *fill_context = (nif_linux_fill_cpu_context *)context; + uint32_t socket_id = 0; + if (!nif_linux_read_socket_id(value, &socket_id)) + return 0; +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c + + fill_context->processor_group_ids[fill_context->index] = 0u; + fill_context->socket_ids[fill_context->index] = socket_id; + fill_context->cpu_indices[fill_context->index] = (uint32_t)value; + fill_context->numa_node_ids[fill_context->index] = (uint32_t)fill_context->numa_id; fill_context->index++; return 1; @@ -211,12 +249,21 @@ static int nfn_linux_populate_writable_mapping(void *address, size_t size) #endif } +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c /* Pin the calling thread to a single logical processor. * core_id >= 0 : bind to that logical processor (global index, across groups). * core_id < 0 : clear affinity constraint within the current group. */ NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id) +======= +/* Linux has no Windows-style processor groups; processor_group_id is ignored. */ +NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id) +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c { - int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN); + (void)processor_group_id; + + int cpu_count = (int)sysconf(_SC_NPROCESSORS_CONF); + if (cpu_index >= cpu_count) + cpu_count = cpu_index + 1; if (cpu_count <= 0) return; @@ -227,11 +274,11 @@ NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id) CPU_ZERO_S(size, set); - if (core_id < 0) { + if (cpu_index < 0) { for (int i = 0; i < cpu_count; i++) CPU_SET_S((int)i, size, set); } else { - CPU_SET_S(core_id, size, set); + CPU_SET_S(cpu_index, size, set); } pthread_setaffinity_np(pthread_self(), size, set); @@ -264,9 +311,18 @@ NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_eac return 1; } +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global) +======= +NIF_API int NIF_DECL nif_get_numa_each(int numa_id, + uint32_t *processor_group_ids, + uint32_t *socket_ids, + uint32_t *cpu_indices, + uint32_t *numa_node_ids) +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c { - if (numa_id < 0) + if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL || + cpu_indices == NULL || numa_node_ids == NULL) return 0; char path[128]; @@ -275,8 +331,20 @@ NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer))) return 0; +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c nfn_linux_fill_cpu_context fill_context = { internal, global, 0 }; return nfn_linux_visit_id_list(buffer, nfn_linux_fill_cpu_id, &fill_context); +======= + nif_linux_fill_cpu_context fill_context = { + processor_group_ids, + socket_ids, + cpu_indices, + numa_node_ids, + numa_id, + 0 + }; + return nif_linux_visit_id_list(buffer, nif_linux_fill_cpu_id, &fill_context); +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c } NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id) diff --git a/src/Nitload.Native/nfn-src/thread.windows.c b/src/Nitload.Native/nfn-src/thread.windows.c index 789bf8fc..d5001882 100644 --- a/src/Nitload.Native/nfn-src/thread.windows.c +++ b/src/Nitload.Native/nfn-src/thread.windows.c @@ -3,33 +3,109 @@ #if NFN_IS_OS(NFN_OS_WINDOWS) +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.windows.c /* Pin the calling thread to a single logical processor. * core_id >= 0 : bind to that logical processor (global index, across groups). * core_id < 0 : clear affinity constraint within the current group. */ NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id) +======= +static int nif_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_group_count) +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.windows.c { - if (core_id < 0 || numa_id < 0) { - /* Query which group the thread currently belongs to */ + DWORD group_count = GetActiveProcessorGroupCount(); + DWORD buffer_size = 0; + uint32_t *socket_map; + SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *information; + BYTE *cursor; + uint32_t socket_id = 0; + + if (group_count == 0) + return 0; + + socket_map = (uint32_t *)malloc((size_t)group_count * 64u * sizeof(uint32_t)); + if (socket_map == NULL) + return 0; + for (size_t i = 0; i < (size_t)group_count * 64u; i++) + socket_map[i] = UINT32_MAX; + + GetLogicalProcessorInformationEx(RelationProcessorPackage, NULL, &buffer_size); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER || buffer_size == 0) { + free(socket_map); + return 0; + } + + information = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)malloc(buffer_size); + if (information == NULL || + !GetLogicalProcessorInformationEx(RelationProcessorPackage, information, &buffer_size)) { + free(information); + free(socket_map); + return 0; + } + + cursor = (BYTE *)information; + while (cursor < (BYTE *)information + buffer_size) { + SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *entry = + (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)cursor; + if (entry->Relationship == RelationProcessorPackage) { + PROCESSOR_RELATIONSHIP *package = &entry->Processor; + for (WORD i = 0; i < package->GroupCount; i++) { + WORD group = package->GroupMask[i].Group; + KAFFINITY mask = package->GroupMask[i].Mask; + if (group >= group_count) + continue; + for (DWORD cpu = 0; cpu < 64; cpu++) { + if ((mask & ((KAFFINITY)1 << cpu)) != 0) + socket_map[(size_t)group * 64u + cpu] = socket_id; + } + } + socket_id++; + } + if (entry->Size == 0) + break; + cursor += entry->Size; + } + + free(information); + *out_socket_map = socket_map; + *out_group_count = group_count; + return 1; +} + +static int nif_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id) +{ + PROCESSOR_NUMBER processor; + memset(&processor, 0, sizeof(processor)); + processor.Group = group; + processor.Number = cpu; + return GetNumaProcessorNodeEx(&processor, node_id) != 0; +} + +/* Pin the calling thread using its processor group and group-local CPU index. */ +NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id) +{ + if (cpu_index < 0 || processor_group_id < 0) { GROUP_AFFINITY current; GetThreadGroupAffinity(GetCurrentThread(), ¤t); - /* Build a full mask for that group's actual core count */ DWORD count = GetActiveProcessorCount(current.Group); - GROUP_AFFINITY ga; - memset(&ga, 0, sizeof(ga)); - ga.Group = current.Group; - ga.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1; - SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL); + GROUP_AFFINITY affinity; + memset(&affinity, 0, sizeof(affinity)); + affinity.Group = current.Group; + affinity.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1; + SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL); return; } - int index = core_id % 64; - DWORD count = GetActiveProcessorCount(numa_id); - if ((DWORD)index >= count) + + if ((DWORD)processor_group_id >= GetActiveProcessorGroupCount() || cpu_index >= 64) return; - GROUP_AFFINITY ga; - memset(&ga, 0, sizeof(ga)); - ga.Group = (WORD)numa_id; - ga.Mask = (KAFFINITY)1 << index; - SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL); + USHORT node_id = 0; + if (!nif_windows_get_processor_node((WORD)processor_group_id, (BYTE)cpu_index, &node_id)) + return; + + GROUP_AFFINITY affinity; + memset(&affinity, 0, sizeof(affinity)); + affinity.Group = (WORD)processor_group_id; + affinity.Mask = (KAFFINITY)1 << cpu_index; + SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL); } NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa) @@ -39,30 +115,64 @@ NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_eac return 0; *out_numa_count = (int)highest + 1; - for (UCHAR n = 0; n <= (UCHAR)highest; n++) { - ULONGLONG mask = 0; - GetNumaNodeProcessorMask(n, &mask); - count_of_each_numa[n] = (int)__popcnt64(mask); + if (count_of_each_numa != NULL) { + for (ULONG n = 0; n <= highest; n++) + count_of_each_numa[n] = 0; + } + + DWORD group_count = GetActiveProcessorGroupCount(); + for (DWORD group = 0; group < group_count; group++) { + for (DWORD cpu = 0; cpu < 64; cpu++) { + USHORT node_id = 0; + if (nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) && + count_of_each_numa != NULL && node_id <= highest) + count_of_each_numa[node_id]++; + } } return 1; } +<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.windows.c NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global) +======= +NIF_API int NIF_DECL nif_get_numa_each(int numa_id, + uint32_t *processor_group_ids, + uint32_t *socket_ids, + uint32_t *cpu_indices, + uint32_t *numa_node_ids) +>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.windows.c { - ULONGLONG mask = 0; - if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask)) + if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL || + cpu_indices == NULL || numa_node_ids == NULL) return 0; - int intra = 0; - while (mask) { - DWORD idx; - _BitScanForward64(&idx, mask); - if (internal != NULL) - internal[intra] = (uint64_t)intra; - global[intra] = (uint32_t)idx; - intra++; - mask &= mask - 1; + uint32_t *socket_map = NULL; + DWORD group_count = 0; + if (!nif_windows_get_socket_map(&socket_map, &group_count)) + return 0; + + int index = 0; + for (DWORD group = 0; group < group_count; group++) { + for (DWORD cpu = 0; cpu < 64; cpu++) { + USHORT node_id = 0; + if (!nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) || + node_id != (USHORT)numa_id) + continue; + + uint32_t socket_id = socket_map[(size_t)group * 64u + cpu]; + if (socket_id == UINT32_MAX) { + free(socket_map); + return 0; + } + processor_group_ids[index] = (uint32_t)group; + socket_ids[index] = socket_id; + cpu_indices[index] = cpu; + numa_node_ids[index] = node_id; + index++; + } } + + free(socket_map); return 1; } diff --git a/src/Nitload.Native/nif-head/NitloadFrameWork.h b/src/Nitload.Native/nif-head/NitloadFrameWork.h new file mode 100644 index 00000000..df1f430f --- /dev/null +++ b/src/Nitload.Native/nif-head/NitloadFrameWork.h @@ -0,0 +1,39 @@ +#ifndef NIF_NATIVE_H +#define NIF_NATIVE_H + +#ifdef __cplusplus +#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage." +#endif + +#include "nif_internal.h" +#include "nif_target.h" + +#include +#include +#include + +#define byte char +#define cs_char uint16_t +#define nif_null_assert(ptr) assert(ptr != NULL) + +#ifndef NIF_ +#endif + +NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table); +NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id); +NIF_API SIMDID NIF_DECL nif_get_simd_level_global(); +/* Per-core variant: always re-executes CPUID, never caches. + * Must be called on the target core (after affinity is set). */ +NIF_API SIMDID NIF_DECL nif_get_simd_level_local(); +NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right); +NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right); +NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id); +NIF_API void* NIF_DECL nif_numa_alloc(size_t size, int numa_id); +NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id); +NIF_API int NIF_DECL nif_get_numa_each(int numa_id, + uint32_t *processor_group_ids, + uint32_t *socket_ids, + uint32_t *cpu_indices, + uint32_t *numa_node_ids); +NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa); +#endif diff --git a/test/STgSharpTest/UnitTest1.cs b/test/STgSharpTest/UnitTest1.cs index 3b842f5b..897eeaec 100644 --- a/test/STgSharpTest/UnitTest1.cs +++ b/test/STgSharpTest/UnitTest1.cs @@ -7,7 +7,7 @@ // // Permission is hereby granted, free of charge, to any person // obtaining a copy of this software and associated documentation -// files (the Software), to deal in the Software without restriction, +// files (the ��Software��), to deal in the Software without restriction, // including without limitation the rights to use, copy, modify, merge, // publish, distribute, sublicense, and/or sell copies of the Software, // and to permit persons to whom the Software is furnished to do so, @@ -17,7 +17,7 @@ // this permission notice shall be included in all copies // or substantial portions of the Software. // -// THE SOFTWARE IS PROVIDED AS IS, +// THE SOFTWARE IS PROVIDED ��AS IS��, // WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, // INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. diff --git a/test/StgSharpDebug/TestSwiss.cs b/test/StgSharpDebug/TestSwiss.cs deleted file mode 100644 index 9b9fd7e2..00000000 --- a/test/StgSharpDebug/TestSwiss.cs +++ /dev/null @@ -1,46 +0,0 @@ -// ----------------------------------------------------------------------------- -// file="TestSwiss" -// Project: StgSharp -// Copyright (c) Nitload. -// SPDX-License-Identifier: MIT -// ----------------------------------------------------------------------------- - -using Nitload.Common.Collections; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; - -namespace Nitload -{ - internal static partial class Program - { - - public static void TestSwiss() - { - SwissTable swiss = SwissTable.Create(); - for (nuint i = 0; i < 5000; i++) - { - Console.Write($"Set {i}: "); - _ = swiss.TryAddOrSet(i, i * 2, out _); - Console.WriteLine($"Set a kvp of ({i},{i * 2})"); - } - Random rand = new Random(); - for (int i = 0; i < 100; i++) - { - Console.Write($"Get {i}: "); - nuint key = (nuint)rand.Next(4000, 5000); - if (swiss.TryGet(key, out nuint value)) - { - Console.WriteLine($"Success in get a kvp of ({key},{value})"); - } else - { - Console.WriteLine($"Failed to get a kvp of ({key},?)"); - } - } - } - - } -}