Implement L4 Swiss map and benchmarks

src/Nitload.Benchmark/Program.cs

@@ -9,16 +9,23 @@ using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
-using BenchmarkDotNet.Toolchains.InProcess.Emit;
 using Nitload.Benchmark;
 
+if (args.Length > 0 && args[0] == "--disasm-swiss")
+{
+    IConfig disassemblyConfig = ManualConfig.Create(DefaultConfig.Instance)
+                                               .AddLogger(ConsoleLogger.Default)
+                                               .AddJob(Job.ShortRun.WithWarmupCount(6));
+    BenchmarkRunner.Run<SwissX86DisassemblyBench>(disassemblyConfig);
+    return;
+}
+
 IConfig config = ManualConfig.Create(DefaultConfig.Instance)
                              .AddLogger(ConsoleLogger.Default)
                              .AddJob(Job.Default
-                                        .WithToolchain(InProcessEmitToolchain.Instance)
                                         .WithLaunchCount(1)
                                         .WithWarmupCount(6)
                                         .WithIterationCount(15)
                                         .WithInvocationCount(3)
                                         .WithUnrollFactor(1));
-_ = BenchmarkRunner.Run<TlsfAllocatorBench>(config);
+BenchmarkSwitcher.FromAssembly(typeof(TlsfAllocatorBench).Assembly).Run(args, config);

src/Nitload.Benchmark/SwissX86Bench.cs

@@ -0,0 +1,306 @@
+// -----------------------------------------------------------------------------
+// file="SwissX86Bench"
+// Project: nFramework
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using Nitload.Common.HighPerformance.Memory;
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Nitload.Benchmark
+{
+    [MemoryDiagnoser]
+    [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)]
+    [CategoriesColumn]
+    public unsafe class SwissX86Bench
+    {
+
+        private const int QueryCount = 1 << 21;
+        private const int ConcurrentQueryCount = QueryCount * 6;
+        private const int MaximumResidentCount = 8192;
+
+        private byte* _buffer;
+        private L4.SwissTable _swiss;
+        private Dictionary<L4.AddressSource, ulong> _dictionary = null!;
+        private ConcurrentDictionary<L4.AddressSource, ulong> _concurrentDictionary = null!;
+        private L4.AddressSource[] _hitQueries = [];
+        private L4.AddressSource[] _missQueries = [];
+        private L4.AddressSource[] _writerQueries = [];
+        private ulong[] _writerValues = [];
+        private byte* _readBuffer;
+        private byte* _writeBuffer;
+        private ParallelOptions _parallelOptions = null!;
+        private Func<ulong> _parallelLocalInit = null!;
+        private Func<int, ParallelLoopState, ulong, ulong> _swissParallelBody = null!;
+        private Func<int, ParallelLoopState, ulong, ulong> _concurrentDictionaryParallelBody = null!;
+        private Action<ulong> _parallelLocalFinally = null!;
+        private long _parallelChecksum;
+
+        [Params(25, 50, 100)]
+        public int LoadPercent { get; set; }
+
+        private int ResidentCount => MaximumResidentCount * LoadPercent / 100;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            if (!Sse42.X64.IsSupported)
+                throw new PlatformNotSupportedException("The Swiss X86 benchmark requires x64 SSE4.2.");
+
+            _buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
+            if (_buffer == null)
+                throw new OutOfMemoryException("Could not allocate Swiss benchmark tables.");
+
+            _swiss = L4.SwissTable.Create(_buffer);
+            _dictionary = new Dictionary<L4.AddressSource, ulong>(ResidentCount);
+            _concurrentDictionary = new ConcurrentDictionary<L4.AddressSource, ulong>(
+                Math.Max(1, Math.Min(Environment.ProcessorCount, 8)),
+                ResidentCount);
+
+            L4.AddressSource[] residentKeys = new L4.AddressSource[ResidentCount];
+            for (int i = 0; i < ResidentCount; i++)
+            {
+                L4.AddressSource key = MakeKey(i);
+                ulong value = (ulong)i + 1;
+                residentKeys[i] = key;
+                if (!_swiss.TryAddOrSet(key, value))
+                    throw new InvalidOperationException($"Swiss insertion failed while seeding key {i}.");
+                _dictionary.Add(key, value);
+                _concurrentDictionary.TryAdd(key, value);
+            }
+
+            _swiss.SwapBuffers();
+            for (int i = 0; i < ResidentCount; i++)
+            {
+                if (!_swiss.TryAddOrSet(residentKeys[i], (ulong)i + 1))
+                    throw new InvalidOperationException($"Swiss writer-buffer seeding failed for key {i}.");
+            }
+
+            _readBuffer = _swiss.ReadBufferForLockFreeBenchmark;
+            _writeBuffer = _swiss.WriteBufferForLockFreeBenchmark;
+
+            _hitQueries = new L4.AddressSource[QueryCount];
+            _missQueries = new L4.AddressSource[QueryCount];
+            _writerQueries = new L4.AddressSource[QueryCount];
+            _writerValues = new ulong[QueryCount];
+            Random random = new(0x51_15_2026);
+            for (int i = 0; i < QueryCount; i++)
+            {
+                _hitQueries[i] = residentKeys[random.Next(ResidentCount)];
+                _missQueries[i] = MakeKey(MaximumResidentCount + i);
+                _writerQueries[i] = residentKeys[random.Next(ResidentCount)];
+                _writerValues[i] = (ulong)i + 0x10000;
+            }
+
+            _parallelOptions = new ParallelOptions
+            {
+                MaxDegreeOfParallelism = Math.Max(2, Math.Min(Environment.ProcessorCount, 8))
+            };
+            _parallelLocalInit = static() => 0UL;
+            _swissParallelBody = ReadSwissParallel;
+            _concurrentDictionaryParallelBody = ReadConcurrentDictionaryParallel;
+            _parallelLocalFinally = MergeParallelChecksum;
+
+            ValidateLookupResults();
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            _swiss.Dispose();
+            NativeMemory.AlignedFree(_buffer);
+            _buffer = null;
+        }
+
+        [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Hit lookup")]
+        public ulong DictionaryHitLookup()
+        {
+            ulong checksum = 0;
+            for (int i = 0; i < QueryCount; i++)
+                checksum += _dictionary[_hitQueries[i]];
+            return checksum;
+        }
+
+        [Benchmark(OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Hit lookup")]
+        public ulong SwissLockFreeHitLookup()
+        {
+            ulong checksum = 0;
+            for (int i = 0; i < QueryCount; i++)
+            {
+                if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong value))
+                    throw new InvalidOperationException("Swiss table missed a seeded hit key.");
+                checksum += value;
+            }
+            return checksum;
+        }
+
+        [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Miss lookup")]
+        public int DictionaryMissLookup()
+        {
+            int misses = 0;
+            for (int i = 0; i < QueryCount; i++)
+                misses += _dictionary.TryGetValue(_missQueries[i], out _) ? 0 : 1;
+            return misses;
+        }
+
+        [Benchmark(OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Miss lookup")]
+        public int SwissLockFreeMissLookup()
+        {
+            int misses = 0;
+            for (int i = 0; i < QueryCount; i++)
+                misses += L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _) ? 0 : 1;
+            return misses;
+        }
+
+        [Benchmark(Baseline = true, OperationsPerInvoke = ConcurrentQueryCount)]
+        [BenchmarkCategory("Concurrent hit lookup")]
+        public ulong ConcurrentDictionaryConcurrentHitLookup()
+        {
+            Interlocked.Exchange(ref _parallelChecksum, 0);
+            Parallel.For(
+                0,
+            ConcurrentQueryCount,
+                _parallelOptions,
+                _parallelLocalInit,
+                _concurrentDictionaryParallelBody,
+                _parallelLocalFinally);
+            return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
+        }
+
+        [Benchmark(OperationsPerInvoke = ConcurrentQueryCount)]
+        [BenchmarkCategory("Concurrent hit lookup")]
+        public ulong SwissLockFreeConcurrentHitLookup()
+        {
+            Interlocked.Exchange(ref _parallelChecksum, 0);
+            Parallel.For(
+                0,
+            ConcurrentQueryCount,
+                _parallelOptions,
+                _parallelLocalInit,
+                _swissParallelBody,
+                _parallelLocalFinally);
+            return unchecked((ulong)Interlocked.Read(ref _parallelChecksum));
+        }
+
+        [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Upsert existing")]
+        public ulong DictionaryUpsertExisting()
+        {
+            for (int i = 0; i < QueryCount; i++)
+                _dictionary[_writerQueries[i]] = _writerValues[i];
+            return _dictionary[_writerQueries[^1]];
+        }
+
+        [Benchmark(OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Upsert existing")]
+        public ulong SwissUpsertExisting()
+        {
+            ulong failures = 0;
+            for (int i = 0; i < QueryCount; i++)
+                failures += L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _writerQueries[i], _writerValues[i]) ? 0UL : 1UL;
+            if (failures != 0 || !L4.SwissReaderX86.TryGet(_writeBuffer, _writerQueries[^1], out ulong value))
+                throw new InvalidOperationException("Swiss upsert failed to preserve the final value.");
+            return value;
+        }
+
+        [Benchmark(Baseline = true, OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Remove and reinsert")]
+        public ulong DictionaryRemoveAndReinsert()
+        {
+            ulong checksum = 0;
+            for (int i = 0; i < QueryCount; i++)
+            {
+                L4.AddressSource key = _writerQueries[i];
+                ulong value = _dictionary[key];
+                _dictionary.Remove(key);
+                _dictionary.Add(key, value + 1);
+                checksum += value;
+            }
+            return checksum;
+        }
+
+        [Benchmark(OperationsPerInvoke = QueryCount)]
+        [BenchmarkCategory("Remove and reinsert")]
+        public ulong SwissRemoveAndReinsert()
+        {
+            ulong checksum = 0;
+            for (int i = 0; i < QueryCount; i++)
+            {
+                L4.AddressSource key = _writerQueries[i];
+                if (!L4.SwissWriterX86.TryRemove(_writeBuffer, key, out ulong value) ||
+                    !L4.SwissWriterX86.TryAddOrSet(_writeBuffer, key, value + 1)) {
+                    throw new InvalidOperationException("Swiss remove/reinsert failed.");
+                }
+                checksum += value;
+            }
+            return checksum;
+        }
+
+        private static L4.AddressSource MakeKey(int index)
+        {
+            return new L4.AddressSource
+            {
+                Address = 0x1000_0000UL + ((ulong)index * 64UL),
+                Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
+            };
+        }
+
+        private ulong ReadSwissParallel(int index, ParallelLoopState state, ulong checksum)
+        {
+            int queryIndex = index & (QueryCount - 1);
+            if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[queryIndex], out ulong value))
+                throw new InvalidOperationException("Swiss table missed a seeded concurrent hit key.");
+            return checksum + value;
+        }
+
+        private ulong ReadConcurrentDictionaryParallel(int index, ParallelLoopState state, ulong checksum)
+        {
+            int queryIndex = index & (QueryCount - 1);
+            if (!_concurrentDictionary.TryGetValue(_hitQueries[queryIndex], out ulong value))
+                throw new InvalidOperationException("ConcurrentDictionary missed a seeded hit key.");
+            return checksum + value;
+        }
+
+        private void MergeParallelChecksum(ulong checksum)
+        {
+            _ = Interlocked.Add(ref _parallelChecksum, unchecked((long)checksum));
+        }
+
+        private void ValidateLookupResults()
+        {
+            for (int i = 0; i < _hitQueries.Length; i++)
+            {
+                bool dictionaryFound = _dictionary.TryGetValue(_hitQueries[i], out ulong dictionaryValue);
+                bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _hitQueries[i], out ulong swissValue);
+                if (!dictionaryFound || !swissFound || dictionaryValue != swissValue)
+                    throw new InvalidOperationException("Swiss and Dictionary hit results differ.");
+
+                if (!_concurrentDictionary.TryGetValue(_hitQueries[i], out ulong concurrentValue) ||
+                    concurrentValue != dictionaryValue)
+                    throw new InvalidOperationException("ConcurrentDictionary and Dictionary hit results differ.");
+            }
+
+            for (int i = 0; i < _missQueries.Length; i++)
+            {
+                bool dictionaryFound = _dictionary.TryGetValue(_missQueries[i], out _);
+                bool swissFound = L4.SwissReaderX86.TryGet(_readBuffer, _missQueries[i], out _);
+                if (dictionaryFound || swissFound)
+                    throw new InvalidOperationException("Swiss and Dictionary miss results differ.");
+            }
+        }
+
+    }
+}

src/Nitload.Benchmark/SwissX86DisassemblyBench.cs

@@ -0,0 +1,135 @@
+// -----------------------------------------------------------------------------
+// file="SwissX86DisassemblyBench"
+// Project: nFramework
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Diagnosers;
+using Nitload.Common.HighPerformance.Memory;
+using System;
+using System.Runtime.InteropServices;
+using System.Runtime.Intrinsics.X86;
+
+namespace Nitload.Benchmark
+{
+    [DisassemblyDiagnoser(maxDepth: 3, exportHtml: true)]
+    public unsafe class SwissX86DisassemblyBench
+    {
+
+        private const int ResidentCount = 8192;
+
+        private byte* _buffer;
+        private L4.SwissTable _table;
+        private byte* _readBuffer;
+        private byte* _writeBuffer;
+        private L4.AddressSource _hitKey;
+        private L4.AddressSource _missKey;
+        private ulong _value;
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            if (!Sse42.X64.IsSupported)
+                throw new PlatformNotSupportedException("The Swiss X86 disassembly probe requires x64 SSE4.2.");
+
+            _buffer = (byte*)NativeMemory.AlignedAlloc((nuint)L4.BufferSize, 64);
+            if (_buffer == null)
+                throw new OutOfMemoryException("Could not allocate Swiss disassembly buffers.");
+
+            _table = L4.SwissTable.Create(_buffer);
+            for (int index = 0; index < ResidentCount; index++)
+            {
+                if (!L4.SwissWriterX86.TryAddOrSet(_table.WriteBufferForLockFreeBenchmark,
+                                                   MakeKey(index), (ulong)index + 1))
+                    throw new InvalidOperationException($"Failed to seed Swiss key {index}.");
+            }
+
+            _table.SwapBuffers();
+            _readBuffer = _table.ReadBufferForLockFreeBenchmark;
+            _writeBuffer = _table.WriteBufferForLockFreeBenchmark;
+            for (int index = 0; index < ResidentCount; index++)
+            {
+                if (!L4.SwissWriterX86.TryAddOrSet(_writeBuffer, MakeKey(index), (ulong)index + 1))
+                    throw new InvalidOperationException($"Failed to seed inactive Swiss key {index}.");
+            }
+
+            _hitKey = MakeKey(ResidentCount / 2);
+            _missKey = MakeKey(ResidentCount + 1);
+            _value = 0x1234_5678;
+
+            if (!L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _) ||
+                L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _))
+                throw new InvalidOperationException("Swiss reader setup validation failed.");
+        }
+
+        [GlobalCleanup]
+        public void Cleanup()
+        {
+            _table.Dispose();
+            NativeMemory.AlignedFree(_buffer);
+            _buffer = null;
+        }
+
+        [Benchmark]
+        public bool RawReaderHit()
+        {
+            return L4.SwissReaderX86.TryGet(_readBuffer, _hitKey, out _value);
+        }
+
+        [Benchmark]
+        public bool RawReaderMiss()
+        {
+            return L4.SwissReaderX86.TryGet(_readBuffer, _missKey, out _value);
+        }
+
+        [Benchmark]
+        public bool RawWriterUpsertExisting()
+        {
+            return L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, _value);
+        }
+
+        [Benchmark]
+        public bool RawWriterRemoveAndReinsert()
+        {
+            return L4.SwissWriterX86.TryRemove(_writeBuffer, _hitKey, out ulong previousValue) &&
+                   L4.SwissWriterX86.TryAddOrSet(_writeBuffer, _hitKey, previousValue);
+        }
+
+        [Benchmark]
+        public bool TableReaderHit()
+        {
+            return _table.TryGet(_hitKey, out _value);
+        }
+
+        [Benchmark]
+        public bool TableReaderMiss()
+        {
+            return _table.TryGet(_missKey, out _value);
+        }
+
+        [Benchmark]
+        public bool TableWriterUpsertExisting()
+        {
+            return _table.TryAddOrSet(_hitKey, _value);
+        }
+
+        [Benchmark]
+        public bool TableWriterRemoveAndReinsert()
+        {
+            return _table.TryRemove(_hitKey, out ulong previousValue) &&
+                   _table.TryAddOrSet(_hitKey, previousValue);
+        }
+
+        private static L4.AddressSource MakeKey(int index)
+        {
+            return new L4.AddressSource
+            {
+                Address = 0x1000_0000UL + ((ulong)index * 64UL),
+                Description = ((ulong)index * 0x9e37_79b9UL) ^ 0xd1b5_4a32_d192_ed03UL
+            };
+        }
+
+    }
+}

src/Nitload.Common/Global.cs

@@ -15,4 +15,5 @@ global using System.Threading;
 [assembly: InternalsVisibleTo("Nitload.RegularAnalysis")]
 [assembly: InternalsVisibleTo("Nitload.Graphics")]
 [assembly: InternalsVisibleTo("Nitload.Mathematics")]
+[assembly: InternalsVisibleTo("Nitload.Benchmark")]
 [assembly: InternalsVisibleTo("StgSharpDebug")]

src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs

@@ -27,6 +27,8 @@ namespace Nitload.Common.HighPerformance.Memory
 
             internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
 
+            [FieldOffset(16)] internal AllocationState State;
+
             public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
 
             #region tlsf linked list
@@ -43,20 +45,27 @@ namespace Nitload.Common.HighPerformance.Memory
             [FieldOffset(8)] public uint PositionMask;
             [FieldOffset(12)] public uint SizeMask;
 
-            #endregion
+        #endregion
-
-            [FieldOffset(16)] internal AllocationState State;
-
         }
 
 #pragma warning disable IDE0032
         [StructLayout(LayoutKind.Explicit, Size = 16)]
-        public struct Handle : IDisposable
+        public unsafe struct Handle : IDisposable
         {
 
             [FieldOffset(0)] private nuint _handle;
+
             [FieldOffset(8)] private nuint _reserved;
 
+            internal Handle(
+                     nuint handle,
+                     nuint reserved
+            )
+            {
+                _handle = handle;
+                _reserved = reserved;
+            }
+
             [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
             public Handle() { }
 

src/Nitload.Common/HighPerformance/Memory/L4.Map.cs

@@ -0,0 +1,98 @@
+// -----------------------------------------------------------------------------
+// file="L4.Map"
+// Project: StgSharp
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Nitload.Common.HighPerformance.Memory
+{
+    public unsafe partial class L4
+    {
+
+        private const int _low_predict = 512;
+
+        private int _predict_count;
+        private int _swiss_reader_counter;
+        private int _swiss_swap_lock;
+        private int _swiss_writer_lock;
+
+        public partial Handle Map(
+                              AddressSource source
+        )
+        {
+            ulong handle;
+            if (Volatile.Read(ref _predict_count) < _low_predict)
+            {
+                if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0)
+                {
+                    try
+                    {
+                        Predict();
+                        _ = Interlocked.Exchange(ref _swiss_swap_lock, 1);
+                        while (Volatile.Read(ref _swiss_reader_counter) != 0) { }
+
+                        _addressMap.SwapBuffers();
+                        if (_addressMap.TryGet(source, out handle)) {
+                            throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
+                        }
+                    }
+                    finally
+                    {
+                        _ = Interlocked.Exchange(ref _swiss_swap_lock, 0);
+                        _ = Interlocked.Exchange(ref _swiss_writer_lock, 0);
+                    }
+                } else if (!TryGetAddress(source, out handle)) {
+                    throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
+                }
+            } else if (!TryGetAddress(source, out handle)) {
+                throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
+            }
+
+            CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
+
+            // process metadata, unfinished
+            uint offset = metadata->PositionMask;
+            nuint address = /*base ptr + */ offset;
+            _ = Interlocked.Decrement(ref _predict_count);
+
+            return new Handle(address, (nuint)handle);
+        }
+
+        private bool TryGetAddress(
+                     AddressSource source,
+                     out ulong handle
+        )
+        {
+            while (true)
+            {
+                while (Volatile.Read(ref _swiss_swap_lock) != 0) { }
+
+                _ = Interlocked.Increment(ref _swiss_reader_counter);
+                if (Volatile.Read(ref _swiss_swap_lock) == 0)
+                {
+                    break;
+                }
+
+                _ = Interlocked.Decrement(ref _swiss_reader_counter);
+            }
+
+            try
+            {
+                return _addressMap.TryGet(source, out handle);
+            }
+            finally
+            {
+                _ = Interlocked.Decrement(ref _swiss_reader_counter);
+            }
+        }
+
+    }
+}

src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs

@@ -0,0 +1,22 @@
+// -----------------------------------------------------------------------------
+// file="L4.Predict"
+// Project: StgSharp
+// Copyright (c) Nitload.
+// SPDX-License-Identifier: MIT
+// -----------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Nitload.Common.HighPerformance.Memory
+{
+    public unsafe partial class L4
+    {
+
+        internal partial void Predict() { }
+
+    }
+}

src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs

@@ -8,7 +8,7 @@
 using System.Buffers;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
-using System.Threading;
+using System.Runtime.Intrinsics.X86;
 
 namespace Nitload.Common.HighPerformance.Memory
 {
@@ -16,8 +16,6 @@ namespace Nitload.Common.HighPerformance.Memory
     {
 
         private const int BucketCapacity = 16;
-        private const uint CrcMagic = 0x7f1c9961;
-
         private const int GroupCount = 1 << GroupCountLog;
         private const int GroupCountLog = 10;
         private const int IdealCount = GroupCount * BucketCapacity;//16384
@@ -36,17 +34,24 @@ namespace Nitload.Common.HighPerformance.Memory
         }
 
         [StructLayout(LayoutKind.Sequential)]
-        internal struct AddressSource
+        public struct AddressSource : IEquatable<AddressSource>
         {
 
             public ulong Address;
             public ulong Description;
 
-            public override readonly bool Equals(
+            public readonly bool Equals(
-                                          object obj
+                                 AddressSource other
             )
             {
-                return obj is AddressSource source && this == source;
+                return this == other;
+            }
+
+            public override readonly bool Equals(
+                                          object? obj
+            )
+            {
+                return obj is AddressSource source && Equals(source);
             }
 
             public override readonly int GetHashCode()
@@ -88,127 +93,102 @@ namespace Nitload.Common.HighPerformance.Memory
 
         }
 
-        internal sealed unsafe class SwissTable : IDisposable
+        internal unsafe struct SwissTable : IDisposable
         {
 
-            private byte* _buffer_read;
             private byte* _buffer_write;
-            private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion);
 
-            private SwissTable(byte* buffers)
+            private ulong _published_buffer;
+
+            private SwissTable(
+                    byte* buffers
+            )
             {
-                _buffer_read = buffers;
+                _published_buffer = (ulong)buffers;
                 _buffer_write = buffers + TableBufferSize;
-                InitializeBuffer(_buffer_read);
+                InitializeBuffer(buffers);
                 InitializeBuffer(_buffer_write);
             }
 
-            internal static SwissTable Create(byte* buffers)
+            internal byte* ReadBufferForLockFreeBenchmark => (byte*)Volatile.Read(ref _published_buffer);
+
+            internal readonly byte* WriteBufferForLockFreeBenchmark => _buffer_write;
+
+            public void Dispose()
+            {
+                _ = Interlocked.Exchange(ref _published_buffer, 0);
+                _buffer_write = null;
+            }
+
+            /// <summary>
+            ///   Publishes the write buffer. The caller must provide the phase barrier around this
+            ///   operation.
+            /// </summary>
+            public void SwapBuffers()
+            {
+                ulong previousBuffer = Interlocked.Exchange(ref _published_buffer, (ulong)_buffer_write);
+                _buffer_write = (byte*)previousBuffer;
+            }
+
+            /// <summary>
+            ///   Writes to the inactive buffer. The caller must serialize all writes.
+            /// </summary>
+            public readonly bool TryAddOrSet(
+                                 AddressSource key,
+                                 ulong value
+            )
+            {
+                if (Sse42.IsSupported) {
+                    return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
+                }
+
+                throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
+            }
+
+            /// <summary>
+            ///   Reads the published buffer. The caller must prevent swaps until all reads finish.
+            /// </summary>
+            public bool TryGet(
+                        AddressSource key,
+                        out ulong value
+            )
+            {
+                byte* publishedBuffer = (byte*)Volatile.Read(ref _published_buffer);
+                if (Sse42.IsSupported) {
+                    return SwissReaderX86.TryGet(publishedBuffer, key, out value);
+                }
+
+                throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
+            }
+
+            /// <summary>
+            ///   Removes from the inactive buffer. The caller must serialize all writes.
+            /// </summary>
+            public bool TryRemove(
+                        AddressSource key,
+                        out ulong value
+            )
+            {
+                if (Sse42.IsSupported) {
+                    return SwissWriterX86.TryRemove(_buffer_write, key, out value);
+                }
+
+                throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
+            }
+
+            internal static SwissTable Create(
+                                       byte* buffers
+            )
             {
                 ArgumentNullException.ThrowIfNull(buffers);
                 return new SwissTable(buffers);
             }
 
-            /// <summary>
+            private static void InitializeBuffer(
-            ///   Reads the published buffer. Callers must prevent a swap while the read is in progress.
+                                byte* buffer
-            /// </summary>
+            )
-            public bool TryGet(AddressSource key, out ulong value)
             {
-                _synchronization.EnterReadLock();
+                Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, IdealCount);
-                try
-                {
-                    return SwissReaderX86.TryGet(_buffer_read, key, out value);
-                }
-                finally
-                {
-                    _synchronization.ExitReadLock();
-                }
-            }
-
-            /// <summary>
-            ///   Writes to the inactive buffer. Calls must be serialized by the caller.
-            /// </summary>
-            public bool TryAddOrSet(AddressSource key, ulong value)
-            {
-                _synchronization.EnterWriteLock();
-                try
-                {
-                    return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
-                }
-                finally
-                {
-                    _synchronization.ExitWriteLock();
-                }
-            }
-
-            /// <summary>
-            ///   Writes to the inactive buffer. Calls must be serialized by the caller.
-            /// </summary>
-            public bool TryRemove(AddressSource key, out ulong value)
-            {
-                _synchronization.EnterWriteLock();
-                try
-                {
-                    return SwissWriterX86.TryRemove(_buffer_write, key, out value);
-                }
-                finally
-                {
-                    _synchronization.ExitWriteLock();
-                }
-            }
-
-            /// <summary>
-            ///   Clears the inactive buffer. Call only after all readers of its previous generation have quiesced.
-            /// </summary>
-            public void PrepareWriteBuffer()
-            {
-                _synchronization.EnterWriteLock();
-                try
-                {
-                    InitializeBuffer(_buffer_write);
-                }
-                finally
-                {
-                    _synchronization.ExitWriteLock();
-                }
-            }
-
-            /// <summary>
-            ///   Publishes the write buffer. The caller must provide the phase barrier around this operation.
-            /// </summary>
-            public void SwapBuffers()
-            {
-                _synchronization.EnterWriteLock();
-                try
-                {
-                    byte* temporary = _buffer_read;
-                    _buffer_read = _buffer_write;
-                    _buffer_write = temporary;
-                }
-                finally
-                {
-                    _synchronization.ExitWriteLock();
-                }
-            }
-
-            public void Dispose()
-            {
-                _synchronization.EnterWriteLock();
-                try
-                {
-                    _buffer_read = null;
-                    _buffer_write = null;
-                }
-                finally
-                {
-                    _synchronization.ExitWriteLock();
-                }
-                _synchronization.Dispose();
-            }
-
-            private static void InitializeBuffer(byte* buffer)
-            {
-                Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount);
             }
 
         }

src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs

@@ -6,6 +6,7 @@
 // -----------------------------------------------------------------------------
 
 using System.Numerics;
+using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
 using System.Runtime.Intrinsics.X86;
 
@@ -14,7 +15,25 @@ namespace Nitload.Common.HighPerformance.Memory
     public partial class L4
     {
 
-        private readonly unsafe struct SwissWriterX86
+        private const uint CrcMagic = 0x7f1c9961;
+
+        internal static class SwissHashX64
+        {
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            public static uint Compute(
+                               AddressSource key
+            )
+            {
+                ulong hash = CrcMagic;
+                hash = Sse42.X64.Crc32(hash, key.Address);
+                hash = Sse42.X64.Crc32(hash, key.Description);
+                return (uint)hash;
+            }
+
+        }
+
+        internal readonly unsafe struct SwissWriterX86
         {
 
             public static bool TryAddOrSet(
@@ -26,7 +45,7 @@ namespace Nitload.Common.HighPerformance.Memory
                 Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
                 SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
 
-                uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
+                uint hash = SwissHashX64.Compute(key);
                 uint begin = hash % GroupCount;
 
                 uint first_tomb_group = 0;
@@ -108,7 +127,7 @@ namespace Nitload.Common.HighPerformance.Memory
             {
                 Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
                 SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
-                uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
+                uint hash = SwissHashX64.Compute(key);
                 uint begin = hash % GroupCount;
                 byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
 
@@ -149,7 +168,7 @@ namespace Nitload.Common.HighPerformance.Memory
 
         }
 
-        private readonly unsafe struct SwissReaderX86
+        internal readonly unsafe struct SwissReaderX86
         {
 
             public static bool TryGet(
@@ -161,7 +180,7 @@ namespace Nitload.Common.HighPerformance.Memory
                 Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
                 SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
 
-                uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
+                uint hash = SwissHashX64.Compute(key);
                 uint begin = hash % GroupCount;
                 byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
 

src/Nitload.Common/HighPerformance/Memory/L4.cs

@@ -54,6 +54,10 @@ namespace Nitload.Common.HighPerformance.Memory
             GC.SuppressFinalize(this);
         }
 
+        public partial Handle Map(
+                              AddressSource source
+        );
+
         internal void Collect()
         {
             ObjectDisposedException.ThrowIf(_disposed, this);
@@ -77,5 +81,7 @@ namespace Nitload.Common.HighPerformance.Memory
             return _bufferAllocator.TryAlloc(size, out handle);
         }
 
+        internal partial void Predict();
+
     }
 }