youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite of swiss, swiss will not be public but l4 specific

Browse Source
This commit is contained in:
Youmu Liu
2026-10-08 18:15:05 +08:00
parent 1845a00390
commit 2e7c3040f4
18 changed files with 367 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Common/Collections/SwissTable.X86.cs
-130
View File
@@ -1,130 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissTable.X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.Collections
{
public partial class SwissTable
{
private unsafe class BucketScannerX86 : IBucketScanner
{
public bool TryFind(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector);
int mask = Sse2.MoveMask(cmpResult);
while (mask != 0)
{
int bitPos = BitOperations.TrailingZeroCount(mask);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= mask - 1;
}
value = default;
return false;
}
public bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
bucket->Control[index] = 0xFE; // mark as DELETED
// no need to remove Key/Value
return true;
}
matches &= matches - 1;
}
value = default;
return false;
}
public bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0)
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128(
(byte*)&bucket->Control);
// === Phase 1: Look up an existing key ===
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
// Key already exists -> update value
bucket->Values[index] = value;
isNewKey = false;
return true;
}
matches &= matches - 1;
}
// === Phase 2: Track an available insertion slot ===
int emptyOrDeleted = Sse2.MoveMask(bucketCtrl);
if (emptyOrDeleted == 0)
{
isNewKey = false;
return false; // bucket full
}
int slot = BitOperations.TrailingZeroCount(emptyOrDeleted);
bucket->Control[slot] = ctrl;
bucket->Keys[slot] = key;
bucket->Values[slot] = value;
isNewKey = true;
return true;
}
}
}
}
src/Nitload.Common/Collections/SwissTable.cs
+52 -151
View File
@@ -5,143 +5,41 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.Collections
{
/// <summary>
///
/// </summary>
public sealed unsafe partial class SwissTable : IDisposable
public sealed unsafe partial class SwissTable
{
private const int BucketCount = 1 << EntryMaskWidth;
private const int BucketSize = 16;
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<Entry>();
private const int GroupCount = 1 << EntryMaskWidth;
public const int MaxCapacity = GroupCount * BucketSize; // 256 entries per bucket, 1024 buckets
private readonly Entry* _entries;
private readonly Action<nuint> _freeHandle;
private bool _disposed;
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<byte>() + (nuint)Unsafe.SizeOf<M128>();
private byte* _controlBytes;
private SwissTable(
Entry* buffer,
Action<nuint> freeHandle
byte* buffer
)
{
_entries = buffer;
_freeHandle = freeHandle;
_controlBytes = buffer;
}
public static SwissTable Create()
{
Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf<Entry>(), 16);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle));
}
private M128* KvpArray => (M128*)(_controlBytes + MaxCapacity);
public static SwissTable Create(
Func<nuint, nuint> allocator,
Action<nuint> freeHandle
private static void Initialize(
byte* buffer
)
{
Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, freeHandle);
}
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_freeHandle((nuint)_entries);
GC.SuppressFinalize(this);
}
public bool TryAddOrSet(
nuint key,
nuint value,
out bool isNewKey
)
{
uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey);
}
public bool TryGet(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryFind(key, hash, bucket, out value);
/*
* hash = CRC32 of key
* entry = low 8 of hash
* ctrl = 8-15 of hash
* index = rest of hash
* find value of hash
*/
}
public bool TryRemove(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryRemove(key, hash, bucket, out value);
}
internal static SwissTable Create(
nuint bufferPtr,
Action<nuint> FreeHandle
)
{
Entry* buffer = (Entry*)bufferPtr;
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, FreeHandle);
}
~SwissTable()
{
Dispose();
}
[InlineArray(BucketCount)]
private struct Entry
{
public Bucket Bucket;
Unsafe.InitBlock(buffer, 0b10000000, BucketSize);
}
private unsafe struct Bucket
@@ -153,45 +51,48 @@ namespace Nitload.Common.Collections
}
private interface IBucketScanner
public static class Reader
{
static IBucketScanner Current { get; } = Create();
bool TryFind(
nuint key,
uint Hash,
Bucket* bucket,
out nuint value
);
unsafe bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
);
bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
);
private static IBucketScanner Create()
public static bool TryGet(
SwissTable table,
nuint key,
out nuint value
)
{
if (Sse42.IsSupported)
{
return new BucketScannerX86();
} else if (ArmBase.IsSupported)
{
throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform.");
} else
{
throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform.");
}
throw new NotImplementedException("SwissTable reader is awaiting a redesigned storage and lookup implementation.");
}
}
public static class Writer
{
public static bool TryAdd(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and insertion implementation.");
}
public static bool TryRemove(
SwissTable table,
nuint key,
out nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and removal implementation.");
}
public static bool TrySet(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and update implementation.");
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs
+2 -1
View File
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -26,7 +27,7 @@ namespace Nitload.Common.HighPerformance.Memory
nuint origin
)
{
return _map.TryGet(origin, out _);
return SwissTable.Reader.TryGet(_map, origin, out _);
}
public int DebugEvict(
src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs
+3 -1
View File
@@ -5,6 +5,8 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
@@ -49,7 +51,7 @@ namespace Nitload.Common.HighPerformance.Memory
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
_ = SwissTable.Writer.TryRemove(_map, head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs
+4 -3
View File
@@ -5,6 +5,7 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System.ComponentModel;
using System.Numerics;
@@ -32,7 +33,7 @@ namespace Nitload.Common.HighPerformance.Memory
{
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!_map.TryGet(address, out _))
if (!SwissTable.Reader.TryGet(_map, address, out _))
{
// miss and record
span[missIndex] = handle;
@@ -47,7 +48,7 @@ namespace Nitload.Common.HighPerformance.Memory
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
if (SwissTable.Reader.TryGet(_map, address, out nuint handle))
{
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
@@ -93,7 +94,7 @@ namespace Nitload.Common.HighPerformance.Memory
// pre fetch them here
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
_ = SwissTable.Writer.TrySet(_map, address, (nuint)head);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
src/Nitload.Common/HighPerformance/Memory/L4.cs
+1 -1
View File
@@ -162,7 +162,7 @@ namespace Nitload.Common.HighPerformance.Memory
}
while (true)
{
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs
+9 -10
View File
@@ -16,22 +16,21 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
public readonly struct LogicalCore
{
public readonly uint GlobalCpuId;
public readonly uint NumaGroupId;
/// <summary>
/// Stays 0 on linux.
/// </summary>
public readonly uint ProcessorGroupId;
public readonly uint SocketId;
public readonly uint CpuIndex;
public readonly uint NumaNodeId;
internal LogicalCore(
uint globalCpuId,
uint numaGroupId,
uint processorGroupId,
uint socketId,
uint cpuIndex,
uint numaNodeId
)
{
GlobalCpuId = globalCpuId;
NumaGroupId = numaGroupId;
ProcessorGroupId = processorGroupId;
SocketId = socketId;
CpuIndex = cpuIndex;
NumaNodeId = numaNodeId;
}
src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs
+49 -17
View File
@@ -33,7 +33,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
return NumaAllocNative(size, numaId);
@@ -69,7 +69,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
NumaFreeNative(address, size, numaId);
@@ -80,8 +80,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
public static partial int GetNumaEach(
uint numaId,
Span<ulong> CpuIndexInNuma,
Span<uint> globalCpuIdInNuma
Span<uint> processorGroupIds,
Span<uint> socketIds,
Span<uint> cpuIndices,
Span<uint> numaNodeIds
);
[LibraryImport(Native.LibName, EntryPoint = "nif_get_numa_overall")]
@@ -97,10 +99,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
L4[] buffers = new L4[NumaNodeCount];
for (int i = 0; i < NumaNodeCount; i++)
L4[] buffers = new L4[_numaNodes.Length];
for (int i = 0; i < _numaNodes.Length; i++)
{
L4 buffer = new L4(i);
L4 buffer = new L4((int)_numaNodes[i].NumaNodeId);
buffers[i] = buffer;
}
return buffers;
@@ -108,14 +110,29 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
internal static void Initialize()
{
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
int numaNodeCount = 0;
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
if (0 == GetNumaOverall(ref numaNodeCount, default)) {
throw new InvalidOperationException("Failed to get NUMA overall information.");
}
_numaNodes = new NumaNode[numaNodeCount];
for (uint i = 0; i < numaNodeCount; i++) {
_numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]);
int[] coreCountInNuma = new int[numaNodeCount];
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
throw new InvalidOperationException("Failed to get NUMA node processor counts.");
}
int activeNodeCount = 0;
for (int i = 0; i < coreCountInNuma.Length; i++) {
if (coreCountInNuma[i] > 0) {
activeNodeCount++;
}
}
NumaNodeCount = activeNodeCount;
_numaNodes = new NumaNode[activeNodeCount];
int nodeIndex = 0;
for (uint numaId = 0; numaId < numaNodeCount; numaId++) {
int coreCount = coreCountInNuma[(int)numaId];
if (coreCount == 0) {
continue;
}
_numaNodes[nodeIndex++] = InitNumaNode(numaId, coreCount);
}
}
@@ -123,8 +140,8 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial void SetThreadAffinity(
int coreId,
int numaId
int cpuIndex,
int processorGroupId
);
[MethodImpl(MethodImplOptions.NoInlining)]
@@ -133,9 +150,24 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
int count
)
{
Span<uint> globalIndex = stackalloc uint[count];
_ = GetNumaEach(numaId, null, globalIndex);
return new(numaId, count, globalIndex);
Span<uint> processorGroupIds = stackalloc uint[count];
Span<uint> socketIds = stackalloc uint[count];
Span<uint> cpuIndices = stackalloc uint[count];
Span<uint> numaNodeIds = stackalloc uint[count];
if (0 == GetNumaEach(numaId, processorGroupIds, socketIds, cpuIndices, numaNodeIds)) {
throw new InvalidOperationException($"Failed to enumerate processors for NUMA node {numaId}.");
}
return new(numaId, count, processorGroupIds, socketIds, cpuIndices, numaNodeIds);
}
private static bool ContainsNode(uint numaId)
{
foreach (NumaNode node in _numaNodes) {
if (node.NumaNodeId == numaId) {
return true;
}
}
return false;
}
[LibraryImport(Native.LibName, EntryPoint = "nif_numa_alloc")]
src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs
+12 -2
View File
@@ -22,16 +22,26 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
internal NumaNode(
uint numaId,
int count,
Span<uint> logicalCoreIds
Span<uint> processorGroupIds,
Span<uint> socketIds,
Span<uint> cpuIndices,
Span<uint> numaNodeIds
)
{
NumaNodeId = numaId;
_logicalCores = new LogicalCore[count];
for (int i = 0; i < count; i++) {
_logicalCores[i] = new LogicalCore((uint)i, numaId, logicalCoreIds[i]);
_logicalCores[i] = new LogicalCore(
processorGroupIds[i],
socketIds[i],
cpuIndices[i],
numaNodeIds[i]);
}
}
public uint NumaNodeId { get; }
public int LogicalCoreCount => _logicalCores.Length;
public ReadOnlySpan<LogicalCore> LogicalCores => _logicalCores;
src/Nitload.Common/Threading/ThreadHelper.cs
+3 -3
View File
@@ -37,7 +37,7 @@ namespace Nitload.Threading
LogicalCore core
)
{
SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId);
SetThreadAffinity(core.CpuIndex, core.ProcessorGroupId);
}
[Conditional("DEBUG")]
@@ -55,8 +55,8 @@ namespace Nitload.Threading
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static partial void SetThreadAffinity(
uint globalCoreId,
uint numaNodeId
uint cpuIndex,
uint processorGroupId
);
#pragma warning restore CA5393
src/Nitload.Common/Timing/StgSharpTime.cs
+1 -1
View File
@@ -97,7 +97,7 @@ namespace Nitload
}
Thread.Sleep(10);
Numa.SetThreadAffinity((int)threadConfig.BindedCoreIndex, (int)threadConfig.BindedNumaNode);
Numa.SetThreadAffinity((int)threadConfig.CpuIndex, (int)threadConfig.ProcessorGroupId);
while (mainProvider.IsRunning)
{
long elapsedTicks;
src/Nitload.Common/Timing/TimeSourceProvider.cs
+4 -4
View File
@@ -14,15 +14,15 @@ using System.Threading;
namespace Nitload.Common.Timing
{
public class TimeSourceStartupConfiguration(
nuint bindedNumaNode = 0,
nuint bindedCoreIndex = 0,
nuint processorGroupId = 0,
nuint cpuIndex = 0,
ThreadPriority threadPriority = ThreadPriority.Highest
)
{
public nuint BindedNumaNode { get; } = bindedNumaNode;
public nuint ProcessorGroupId { get; } = processorGroupId;
public nuint BindedCoreIndex { get; } = bindedCoreIndex;
public nuint CpuIndex { get; } = cpuIndex;
public ThreadPriority Priority { get; } = threadPriority;