diff --git a/src/Nitload.Benchmark/Program.cs b/src/Nitload.Benchmark/Program.cs index 43c14f8e..129310a8 100644 --- a/src/Nitload.Benchmark/Program.cs +++ b/src/Nitload.Benchmark/Program.cs @@ -9,16 +9,23 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.InProcess.Emit; using Nitload.Benchmark; +if (args.Length > 0 && args[0] == "--disasm-swiss") +{ + IConfig disassemblyConfig = ManualConfig.Create(DefaultConfig.Instance) + .AddLogger(ConsoleLogger.Default) + .AddJob(Job.ShortRun.WithWarmupCount(6)); + BenchmarkRunner.Run(disassemblyConfig); + return; +} + IConfig config = ManualConfig.Create(DefaultConfig.Instance) .AddLogger(ConsoleLogger.Default) .AddJob(Job.Default - .WithToolchain(InProcessEmitToolchain.Instance) .WithLaunchCount(1) .WithWarmupCount(6) .WithIterationCount(15) .WithInvocationCount(3) .WithUnrollFactor(1)); -_ = BenchmarkRunner.Run(config); +BenchmarkSwitcher.FromAssembly(typeof(TlsfAllocatorBench).Assembly).Run(args, config); diff --git a/src/Nitload.Benchmark/SwissX86Bench.cs b/src/Nitload.Benchmark/SwissX86Bench.cs new file mode 100644 index 00000000..29eac81a --- /dev/null +++ b/src/Nitload.Benchmark/SwissX86Bench.cs @@ -0,0 +1,306 @@ +// ----------------------------------------------------------------------------- +// file="SwissX86Bench" +// Project: nFramework +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using Nitload.Common.HighPerformance.Memory; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; +using System.Threading; +using System.Threading.Tasks; + +namespace Nitload.Benchmark +{ + [MemoryDiagnoser] + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] + [CategoriesColumn] + public unsafe class SwissX86Bench + { + + private const int QueryCount = 1 << 21; + private const int ConcurrentQueryCount = QueryCount * 6; + private const int MaximumResidentCount = 8192; + + private byte* _buffer; + private L4.SwissTable _swiss; + private Dictionary _dictionary = null!; + private ConcurrentDictionary _concurrentDictionary = null!; + private L4.AddressSource[] _hitQueries = []; + private L4.AddressSource[] _missQueries = []; + private L4.AddressSource[] _writerQueries = []; + private ulong[] _writerValues = []; + private byte* _readBuffer; + private byte* _writeBuffer; + private ParallelOptions _parallelOptions = null!; + private Func _parallelLocalInit = null!; + private Func _swissParallelBody = null!; + private Func _concurrentDictionaryParallelBody = null!; + private Action _parallelLocalFinally = null!; + private long _parallelChecksum; + + [Params(25, 50, 100)] + public int LoadPercent { get; set; } + + private int ResidentCount => MaximumResidentCount * LoadPercent / 100; + + [GlobalSetup] + public void Setup() + { + if (!Sse42.X64.IsSupported) + throw new PlatformNotSupportedException("The Swiss X86 benchmark requires x64 SSE4.2."); + + _buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64); + if (_buffer == null) + throw new OutOfMemoryException("Could not allocate Swiss benchmark tables."); + + _swiss = L4.SwissTable.Create(_buffer); + _dictionary = new Dictionary(ResidentCount); + _concurrentDictionary = new ConcurrentDictionary( + Math.Max(1, Math.Min(Environment.ProcessorCount, 8)), + ResidentCount); + + L4.AddressSource[] residentKeys = new L4.AddressSource[ResidentCount]; + for (int i = 0; i < ResidentCount; i++) + { + L4.AddressSource key = MakeKey(i); + ulong value = (ulong)i + 1; + residentKeys[i] = key; + if (!_swiss.TryAddOrSet(key, value)) + throw new InvalidOperationException($"Swiss insertion failed while seeding key {i}."); + _dictionary.Add(key, value); + _concurrentDictionary.TryAdd(key, value); + } + + _swiss.SwapBuffers(); + for (int i = 0; i < ResidentCount; i++) + { + if (!_swiss.TryAddOrSet(residentKeys[i], (ulong)i + 1)) + throw new InvalidOperationException($"Swiss writer-buffer seeding failed for key {i}."); + } + + _readBuffer = _swiss.ReadBufferForLockFreeBenchmark; + _writeBuffer = _swiss.WriteBufferForLockFreeBenchmark; + + _hitQueries = new L4.AddressSource[QueryCount]; + _missQueries = new L4.AddressSource[QueryCount]; + _writerQueries = new L4.AddressSource[QueryCount]; + _writerValues = new ulong[QueryCount]; + Random random = new(0x51_15_2026); + for (int i = 0; i < QueryCount; i++) + { + _hitQueries[i] = residentKeys[random.Next(ResidentCount)]; + _missQueries[i] = MakeKey(MaximumResidentCount + i); + _writerQueries[i] = residentKeys[random.Next(ResidentCount)]; + _writerValues[i] = (ulong)i + 0x10000; + } + + _parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = Math.Max(2, Math.Min(Environment.ProcessorCount, 8)) + }; + _parallelLocalInit = static() => 0UL; + _swissParallelBody = ReadSwissParallel; + _concurrentDictionaryParallelBody = ReadConcurrentDictionaryParallel; + _parallelLocalFinally = MergeParallelChecksum; + + ValidateLookupResults(); + } + + [GlobalCleanup] + public void Cleanup() + { + _swiss.Dispose(); + NativeMemory.AlignedFree(_buffer); + _buffer = null; + } + + [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Hit lookup")] + public ulong DictionaryHitLookup() + { + ulong checksum = 0; + for (int i = 0; i < QueryCount; i++) + checksum += _dictionary[_hitQueries[i]]; + return checksum; + } + + [Benchmark(OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Hit lookup")] + public ulong SwissLockFreeHitLookup() + { + ulong checksum = 0; + for (int i = 0; i < QueryCount; i++) + { + if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong value)) + throw new InvalidOperationException("Swiss table missed a seeded hit key."); + checksum += value; + } + return checksum; + } + + [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Miss lookup")] + public int DictionaryMissLookup() + { + int misses = 0; + for (int i = 0; i < QueryCount; i++) + misses += _dictionary.TryGetValue(_missQueries[i], out _) ? 0 : 1; + return misses; + } + + [Benchmark(OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Miss lookup")] + public int SwissLockFreeMissLookup() + { + int misses = 0; + for (int i = 0; i < QueryCount; i++) + misses += L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _) ? 0 : 1; + return misses; + } + + [Benchmark(Baseline = true, OperationsPerInvoke = ConcurrentQueryCount)] + [BenchmarkCategory("Concurrent hit lookup")] + public ulong ConcurrentDictionaryConcurrentHitLookup() + { + Interlocked.Exchange(ref _parallelChecksum, 0); + Parallel.For( + 0, + ConcurrentQueryCount, + _parallelOptions, + _parallelLocalInit, + _concurrentDictionaryParallelBody, + _parallelLocalFinally); + return unchecked((ulong)Interlocked.Read(ref _parallelChecksum)); + } + + [Benchmark(OperationsPerInvoke = ConcurrentQueryCount)] + [BenchmarkCategory("Concurrent hit lookup")] + public ulong SwissLockFreeConcurrentHitLookup() + { + Interlocked.Exchange(ref _parallelChecksum, 0); + Parallel.For( + 0, + ConcurrentQueryCount, + _parallelOptions, + _parallelLocalInit, + _swissParallelBody, + _parallelLocalFinally); + return unchecked((ulong)Interlocked.Read(ref _parallelChecksum)); + } + + [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Upsert existing")] + public ulong DictionaryUpsertExisting() + { + for (int i = 0; i < QueryCount; i++) + _dictionary[_writerQueries[i]] = _writerValues[i]; + return _dictionary[_writerQueries[^1]]; + } + + [Benchmark(OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Upsert existing")] + public ulong SwissUpsertExisting() + { + ulong failures = 0; + for (int i = 0; i < QueryCount; i++) + failures += L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _writerQueries[i], _writerValues[i]) ? 0UL : 1UL; + if (failures != 0 || !L4.SwissReaderX86.TryGet(_writeBuffer, _writerQueries[^1], out ulong value)) + throw new InvalidOperationException("Swiss upsert failed to preserve the final value."); + return value; + } + + [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Remove and reinsert")] + public ulong DictionaryRemoveAndReinsert() + { + ulong checksum = 0; + for (int i = 0; i < QueryCount; i++) + { + L4.AddressSource key = _writerQueries[i]; + ulong value = _dictionary[key]; + _dictionary.Remove(key); + _dictionary.Add(key, value + 1); + checksum += value; + } + return checksum; + } + + [Benchmark(OperationsPerInvoke = QueryCount)] + [BenchmarkCategory("Remove and reinsert")] + public ulong SwissRemoveAndReinsert() + { + ulong checksum = 0; + for (int i = 0; i < QueryCount; i++) + { + L4.AddressSource key = _writerQueries[i]; + if (!L4.SwissWriterX86.TryRemove(_writeBuffer, key, out ulong value) || + !L4.SwissWriterX86.TryAddOrSet(_writeBuffer, key, value + 1)) { + throw new InvalidOperationException("Swiss remove/reinsert failed."); + } + checksum += value; + } + return checksum; + } + + private static L4.AddressSource MakeKey(int index) + { + return new L4.AddressSource + { + Address = 0x1000_0000UL + ((ulong)index * 64UL), + Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL + }; + } + + private ulong ReadSwissParallel(int index, ParallelLoopState state, ulong checksum) + { + int queryIndex = index & (QueryCount - 1); + if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[queryIndex], out ulong value)) + throw new InvalidOperationException("Swiss table missed a seeded concurrent hit key."); + return checksum + value; + } + + private ulong ReadConcurrentDictionaryParallel(int index, ParallelLoopState state, ulong checksum) + { + int queryIndex = index & (QueryCount - 1); + if (!_concurrentDictionary.TryGetValue(_hitQueries[queryIndex], out ulong value)) + throw new InvalidOperationException("ConcurrentDictionary missed a seeded hit key."); + return checksum + value; + } + + private void MergeParallelChecksum(ulong checksum) + { + _ = Interlocked.Add(ref _parallelChecksum, unchecked((long)checksum)); + } + + private void ValidateLookupResults() + { + for (int i = 0; i < _hitQueries.Length; i++) + { + bool dictionaryFound = _dictionary.TryGetValue(_hitQueries[i], out ulong dictionaryValue); + bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong swissValue); + if (!dictionaryFound || !swissFound || dictionaryValue != swissValue) + throw new InvalidOperationException("Swiss and Dictionary hit results differ."); + + if (!_concurrentDictionary.TryGetValue(_hitQueries[i], out ulong concurrentValue) || + concurrentValue != dictionaryValue) + throw new InvalidOperationException("ConcurrentDictionary and Dictionary hit results differ."); + } + + for (int i = 0; i < _missQueries.Length; i++) + { + bool dictionaryFound = _dictionary.TryGetValue(_missQueries[i], out _); + bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _); + if (dictionaryFound || swissFound) + throw new InvalidOperationException("Swiss and Dictionary miss results differ."); + } + } + + } +} \ No newline at end of file diff --git a/src/Nitload.Benchmark/SwissX86DisassemblyBench.cs b/src/Nitload.Benchmark/SwissX86DisassemblyBench.cs new file mode 100644 index 00000000..2461abd4 --- /dev/null +++ b/src/Nitload.Benchmark/SwissX86DisassemblyBench.cs @@ -0,0 +1,135 @@ +// ----------------------------------------------------------------------------- +// file="SwissX86DisassemblyBench" +// Project: nFramework +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Diagnosers; +using Nitload.Common.HighPerformance.Memory; +using System; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace Nitload.Benchmark +{ + [DisassemblyDiagnoser(maxDepth: 3, exportHtml: true)] + public unsafe class SwissX86DisassemblyBench + { + + private const int ResidentCount = 8192; + + private byte* _buffer; + private L4.SwissTable _table; + private byte* _readBuffer; + private byte* _writeBuffer; + private L4.AddressSource _hitKey; + private L4.AddressSource _missKey; + private ulong _value; + + [GlobalSetup] + public void Setup() + { + if (!Sse42.X64.IsSupported) + throw new PlatformNotSupportedException("The Swiss X86 disassembly probe requires x64 SSE4.2."); + + _buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64); + if (_buffer == null) + throw new OutOfMemoryException("Could not allocate Swiss disassembly buffers."); + + _table = L4.SwissTable.Create(_buffer); + for (int index = 0; index < ResidentCount; index++) + { + if (!L4.SwissWriterX86.TryAddOrSet(_table.WriteBufferForLockFreeBenchmark, + MakeKey(index), (ulong)index + 1)) + throw new InvalidOperationException($"Failed to seed Swiss key {index}."); + } + + _table.SwapBuffers(); + _readBuffer = _table.ReadBufferForLockFreeBenchmark; + _writeBuffer = _table.WriteBufferForLockFreeBenchmark; + for (int index = 0; index < ResidentCount; index++) + { + if (!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, MakeKey(index), (ulong)index + 1)) + throw new InvalidOperationException($"Failed to seed inactive Swiss key {index}."); + } + + _hitKey = MakeKey(ResidentCount / 2); + _missKey = MakeKey(ResidentCount + 1); + _value = 0x1234_5678; + + if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _) || + L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _)) + throw new InvalidOperationException("Swiss reader setup validation failed."); + } + + [GlobalCleanup] + public void Cleanup() + { + _table.Dispose(); + NativeMemory.AlignedFree(_buffer); + _buffer = null; + } + + [Benchmark] + public bool RawReaderHit() + { + return L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _value); + } + + [Benchmark] + public bool RawReaderMiss() + { + return L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _value); + } + + [Benchmark] + public bool RawWriterUpsertExisting() + { + return L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, _value); + } + + [Benchmark] + public bool RawWriterRemoveAndReinsert() + { + return L4.SwissWriterX86.TryRemove(_writeBuffer, _hitKey, out ulong previousValue) && + L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, previousValue); + } + + [Benchmark] + public bool TableReaderHit() + { + return _table.TryGet(_hitKey, out _value); + } + + [Benchmark] + public bool TableReaderMiss() + { + return _table.TryGet(_missKey, out _value); + } + + [Benchmark] + public bool TableWriterUpsertExisting() + { + return _table.TryAddOrSet(_hitKey, _value); + } + + [Benchmark] + public bool TableWriterRemoveAndReinsert() + { + return _table.TryRemove(_hitKey, out ulong previousValue) && + _table.TryAddOrSet(_hitKey, previousValue); + } + + private static L4.AddressSource MakeKey(int index) + { + return new L4.AddressSource + { + Address = 0x1000_0000UL + ((ulong)index * 64UL), + Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL + }; + } + + } +} \ No newline at end of file diff --git a/src/Nitload.Common/Global.cs b/src/Nitload.Common/Global.cs index be87319d..d8a196f4 100644 --- a/src/Nitload.Common/Global.cs +++ b/src/Nitload.Common/Global.cs @@ -15,4 +15,5 @@ global using System.Threading; [assembly: InternalsVisibleTo("Nitload.RegularAnalysis")] [assembly: InternalsVisibleTo("Nitload.Graphics")] [assembly: InternalsVisibleTo("Nitload.Mathematics")] +[assembly: InternalsVisibleTo("Nitload.Benchmark")] [assembly: InternalsVisibleTo("StgSharpDebug")] diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs index 7c605e60..5d03dedb 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -27,6 +27,8 @@ namespace Nitload.Common.HighPerformance.Memory internal const uint MaxBufferSize = 0xFFFFFFFF - 64U; + [FieldOffset(16)] internal AllocationState State; + public static uint SizeOf { get; } = (uint)Unsafe.SizeOf(); #region tlsf linked list @@ -43,20 +45,27 @@ namespace Nitload.Common.HighPerformance.Memory [FieldOffset(8)] public uint PositionMask; [FieldOffset(12)] public uint SizeMask; - #endregion - - [FieldOffset(16)] internal AllocationState State; - + #endregion } #pragma warning disable IDE0032 [StructLayout(LayoutKind.Explicit, Size = 16)] - public struct Handle : IDisposable + public unsafe struct Handle : IDisposable { [FieldOffset(0)] private nuint _handle; + [FieldOffset(8)] private nuint _reserved; + internal Handle( + nuint handle, + nuint reserved + ) + { + _handle = handle; + _reserved = reserved; + } + [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] public Handle() { } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs new file mode 100644 index 00000000..0b718658 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Map.cs @@ -0,0 +1,98 @@ +// ----------------------------------------------------------------------------- +// file="L4.Map" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Nitload.Common.HighPerformance.Memory +{ + public unsafe partial class L4 + { + + private const int _low_predict = 512; + + private int _predict_count; + private int _swiss_reader_counter; + private int _swiss_swap_lock; + private int _swiss_writer_lock; + + public partial Handle Map( + AddressSource source + ) + { + ulong handle; + if (Volatile.Read(ref _predict_count) < _low_predict) + { + if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0) + { + try + { + Predict(); + _ = Interlocked.Exchange(ref _swiss_swap_lock, 1); + while (Volatile.Read(ref _swiss_reader_counter) != 0) { } + + _addressMap.SwapBuffers(); + if (_addressMap.TryGet(source, out handle)) { + throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); + } + } + finally + { + _ = Interlocked.Exchange(ref _swiss_swap_lock, 0); + _ = Interlocked.Exchange(ref _swiss_writer_lock, 0); + } + } else if (!TryGetAddress(source, out handle)) { + throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); + } + } else if (!TryGetAddress(source, out handle)) { + throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}]."); + } + + CacheLineMetadata* metadata = (CacheLineMetadata*)handle; + + // process metadata, unfinished + uint offset = metadata->PositionMask; + nuint address = /*base ptr + */ offset; + _ = Interlocked.Decrement(ref _predict_count); + + return new Handle(address, (nuint)handle); + } + + private bool TryGetAddress( + AddressSource source, + out ulong handle + ) + { + while (true) + { + while (Volatile.Read(ref _swiss_swap_lock) != 0) { } + + _ = Interlocked.Increment(ref _swiss_reader_counter); + if (Volatile.Read(ref _swiss_swap_lock) == 0) + { + break; + } + + _ = Interlocked.Decrement(ref _swiss_reader_counter); + } + + try + { + return _addressMap.TryGet(source, out handle); + } + finally + { + _ = Interlocked.Decrement(ref _swiss_reader_counter); + } + } + + } +} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs new file mode 100644 index 00000000..333fe530 --- /dev/null +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs @@ -0,0 +1,22 @@ +// ----------------------------------------------------------------------------- +// file="L4.Predict" +// Project: StgSharp +// Copyright (c) Nitload. +// SPDX-License-Identifier: MIT +// ----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Nitload.Common.HighPerformance.Memory +{ + public unsafe partial class L4 + { + + internal partial void Predict() { } + + } +} diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs index f87d40bd..42c6decf 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs @@ -8,7 +8,7 @@ using System.Buffers; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Threading; +using System.Runtime.Intrinsics.X86; namespace Nitload.Common.HighPerformance.Memory { @@ -16,8 +16,6 @@ namespace Nitload.Common.HighPerformance.Memory { private const int BucketCapacity = 16; - private const uint CrcMagic = 0x7f1c9961; - private const int GroupCount = 1 << GroupCountLog; private const int GroupCountLog = 10; private const int IdealCount = GroupCount * BucketCapacity;//16384 @@ -36,17 +34,24 @@ namespace Nitload.Common.HighPerformance.Memory } [StructLayout(LayoutKind.Sequential)] - internal struct AddressSource + public struct AddressSource : IEquatable { public ulong Address; public ulong Description; - public override readonly bool Equals( - object obj + public readonly bool Equals( + AddressSource other ) { - return obj is AddressSource source && this == source; + return this == other; + } + + public override readonly bool Equals( + object? obj + ) + { + return obj is AddressSource source && Equals(source); } public override readonly int GetHashCode() @@ -88,127 +93,102 @@ namespace Nitload.Common.HighPerformance.Memory } - internal sealed unsafe class SwissTable : IDisposable + internal unsafe struct SwissTable : IDisposable { - private byte* _buffer_read; private byte* _buffer_write; - private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion); - private SwissTable(byte* buffers) + private ulong _published_buffer; + + private SwissTable( + byte* buffers + ) { - _buffer_read = buffers; + _published_buffer = (ulong)buffers; _buffer_write = buffers + TableBufferSize; - InitializeBuffer(_buffer_read); + InitializeBuffer(buffers); InitializeBuffer(_buffer_write); } - internal static SwissTable Create(byte* buffers) + internal byte* ReadBufferForLockFreeBenchmark => (byte*)Volatile.Read(ref _published_buffer); + + internal readonly byte* WriteBufferForLockFreeBenchmark => _buffer_write; + + public void Dispose() + { + _ = Interlocked.Exchange(ref _published_buffer, 0); + _buffer_write = null; + } + + /// + /// Publishes the write buffer. The caller must provide the phase barrier around this + /// operation. + /// + public void SwapBuffers() + { + ulong previousBuffer = Interlocked.Exchange(ref _published_buffer, (ulong)_buffer_write); + _buffer_write = (byte*)previousBuffer; + } + + /// + /// Writes to the inactive buffer. The caller must serialize all writes. + /// + public readonly bool TryAddOrSet( + AddressSource key, + ulong value + ) + { + if (Sse42.IsSupported) { + return SwissWriterX86.TryAddOrSet(_buffer_write, key, value); + } + + throw new NotImplementedException("The Swiss backend is not implemented for this ISA."); + } + + /// + /// Reads the published buffer. The caller must prevent swaps until all reads finish. + /// + public bool TryGet( + AddressSource key, + out ulong value + ) + { + byte* publishedBuffer = (byte*)Volatile.Read(ref _published_buffer); + if (Sse42.IsSupported) { + return SwissReaderX86.TryGet(publishedBuffer, key, out value); + } + + throw new NotImplementedException("The Swiss backend is not implemented for this ISA."); + } + + /// + /// Removes from the inactive buffer. The caller must serialize all writes. + /// + public bool TryRemove( + AddressSource key, + out ulong value + ) + { + if (Sse42.IsSupported) { + return SwissWriterX86.TryRemove(_buffer_write, key, out value); + } + + throw new NotImplementedException("The Swiss backend is not implemented for this ISA."); + } + + internal static SwissTable Create( + byte* buffers + ) { ArgumentNullException.ThrowIfNull(buffers); return new SwissTable(buffers); } - /// - /// Reads the published buffer. Callers must prevent a swap while the read is in progress. - /// - public bool TryGet(AddressSource key, out ulong value) + private static void InitializeBuffer( + byte* buffer + ) { - _synchronization.EnterReadLock(); - try - { - return SwissReaderX86.TryGet(_buffer_read, key, out value); - } - finally - { - _synchronization.ExitReadLock(); - } - } - - /// - /// Writes to the inactive buffer. Calls must be serialized by the caller. - /// - public bool TryAddOrSet(AddressSource key, ulong value) - { - _synchronization.EnterWriteLock(); - try - { - return SwissWriterX86.TryAddOrSet(_buffer_write, key, value); - } - finally - { - _synchronization.ExitWriteLock(); - } - } - - /// - /// Writes to the inactive buffer. Calls must be serialized by the caller. - /// - public bool TryRemove(AddressSource key, out ulong value) - { - _synchronization.EnterWriteLock(); - try - { - return SwissWriterX86.TryRemove(_buffer_write, key, out value); - } - finally - { - _synchronization.ExitWriteLock(); - } - } - - /// - /// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced. - /// - public void PrepareWriteBuffer() - { - _synchronization.EnterWriteLock(); - try - { - InitializeBuffer(_buffer_write); - } - finally - { - _synchronization.ExitWriteLock(); - } - } - - /// - /// Publishes the write buffer. The caller must provide the phase barrier around this operation. - /// - public void SwapBuffers() - { - _synchronization.EnterWriteLock(); - try - { - byte* temporary = _buffer_read; - _buffer_read = _buffer_write; - _buffer_write = temporary; - } - finally - { - _synchronization.ExitWriteLock(); - } - } - - public void Dispose() - { - _synchronization.EnterWriteLock(); - try - { - _buffer_read = null; - _buffer_write = null; - } - finally - { - _synchronization.ExitWriteLock(); - } - _synchronization.Dispose(); - } - - private static void InitializeBuffer(byte* buffer) - { - Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount); + Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, IdealCount); } } diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs index a92933c1..f2edc6b2 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs @@ -6,6 +6,7 @@ // ----------------------------------------------------------------------------- using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; @@ -14,7 +15,25 @@ namespace Nitload.Common.HighPerformance.Memory public partial class L4 { - private readonly unsafe struct SwissWriterX86 + private const uint CrcMagic = 0x7f1c9961; + + internal static class SwissHashX64 + { + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static uint Compute( + AddressSource key + ) + { + ulong hash = CrcMagic; + hash = Sse42.X64.Crc32(hash, key.Address); + hash = Sse42.X64.Crc32(hash, key.Description); + return (uint)hash; + } + + } + + internal readonly unsafe struct SwissWriterX86 { public static bool TryAddOrSet( @@ -26,7 +45,7 @@ namespace Nitload.Common.HighPerformance.Memory Vector128* control_bytes = (Vector128*)buffer; SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); - uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint hash = SwissHashX64.Compute(key); uint begin = hash % GroupCount; uint first_tomb_group = 0; @@ -108,7 +127,7 @@ namespace Nitload.Common.HighPerformance.Memory { Vector128* control_bytes = (Vector128*)buffer; SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); - uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint hash = SwissHashX64.Compute(key); uint begin = hash % GroupCount; byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); @@ -149,7 +168,7 @@ namespace Nitload.Common.HighPerformance.Memory } - private readonly unsafe struct SwissReaderX86 + internal readonly unsafe struct SwissReaderX86 { public static bool TryGet( @@ -161,7 +180,7 @@ namespace Nitload.Common.HighPerformance.Memory Vector128* control_bytes = (Vector128*)buffer; SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount); - uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode()); + uint hash = SwissHashX64.Compute(key); uint begin = hash % GroupCount; byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7)); diff --git a/src/Nitload.Common/HighPerformance/Memory/L4.cs b/src/Nitload.Common/HighPerformance/Memory/L4.cs index e0c30f3f..54ec138d 100644 --- a/src/Nitload.Common/HighPerformance/Memory/L4.cs +++ b/src/Nitload.Common/HighPerformance/Memory/L4.cs @@ -54,6 +54,10 @@ namespace Nitload.Common.HighPerformance.Memory GC.SuppressFinalize(this); } + public partial Handle Map( + AddressSource source + ); + internal void Collect() { ObjectDisposedException.ThrowIf(_disposed, this); @@ -77,5 +81,7 @@ namespace Nitload.Common.HighPerformance.Memory return _bufferAllocator.TryAlloc(size, out handle); } + internal partial void Predict(); + } }