youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

swiss x86 is briefly ok, ready for test

Browse Source
This commit is contained in:
Youmu Liu
2026-10-09 18:17:03 +08:00
parent 2e7c3040f4
commit 3a2b123478
15 changed files with 457 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Common/Collections/SwissTable.cs
-125
View File
@@ -1,125 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Nitload.Common.Collections
{
public sealed unsafe partial class SwissTable
{
private const int BucketSize = 16;
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
private const int GroupCount = 1 << EntryMaskWidth;
public const int MaxCapacity = GroupCount * BucketSize; // 256 entries per bucket, 1024 buckets
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<byte>() + (nuint)Unsafe.SizeOf<M128>();
private byte* _controlBytes;
private SwissTable(
byte* buffer
)
{
_controlBytes = buffer;
}
private M128* KvpArray => (M128*)(_controlBytes + MaxCapacity);
private static void Initialize(
byte* buffer
)
{
Unsafe.InitBlock(buffer, 0b10000000, BucketSize);
}
private unsafe struct Bucket
{
public BucketControl Control;
public BucketKeyStorage Keys;
public BucketValueStorage Values;
}
public static class Reader
{
public static bool TryGet(
SwissTable table,
nuint key,
out nuint value
)
{
throw new NotImplementedException("SwissTable reader is awaiting a redesigned storage and lookup implementation.");
}
}
public static class Writer
{
public static bool TryAdd(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and insertion implementation.");
}
public static bool TryRemove(
SwissTable table,
nuint key,
out nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and removal implementation.");
}
public static bool TrySet(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and update implementation.");
}
}
[InlineArray(BucketSize)]
private struct BucketControl
{
public byte Control;
}
[InlineArray(BucketSize)]
private struct BucketKeyStorage
{
public nuint value;
}
[InlineArray(BucketSize)]
private struct BucketValueStorage
{
public nuint value;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs
+5 -25
View File
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory
@@ -23,15 +22,11 @@ namespace Nitload.Common.HighPerformance.Memory
}
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata
private struct CacheLineMetadata
{
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(24)] internal nuint Origin;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
@@ -50,32 +45,17 @@ namespace Nitload.Common.HighPerformance.Memory
#endregion
#region mask
[FieldOffset(16)] internal AllocationState State;
[FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion
}
#pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit)]
public unsafe struct Handle : IDisposable
[StructLayout(LayoutKind.Explicit, Size = 16)]
public struct Handle : IDisposable
{
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
[FieldOffset(0)] private nuint _handle;
private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{
_handle = handle;
_head = cacheRefCount;
}
[FieldOffset(8)] private nuint _reserved;
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
public Handle() { }
@@ -88,7 +68,7 @@ namespace Nitload.Common.HighPerformance.Memory
return;
}
_handle = 0;
_ = Interlocked.Decrement(ref _head->RefCount);
_reserved = 0;
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs
-93
View File
@@ -1,93 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Debug"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
/**/
public partial class L4
{
public void DebugAgeAll()
{
// AgeAll();
}
public bool DebugContains(
nuint origin
)
{
return SwissTable.Reader.TryGet(_map, origin, out _);
}
public int DebugEvict(
int count = 1
)
{
return Evict(count);
}
public unsafe DebugSnapshot GetDebugSnapshot()
{
throw new NotImplementedException();
/*
int maxId = _maxId;
byte[] rrpv = new byte[maxId];
byte* p = (byte*)_eviction;
for (int i = 0; i < maxId; i++) {
rrpv[i] = p[i];
}
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
/**/
}
public readonly struct DebugSnapshot
{
/// <summary>
/// RRPV value for each line（0~3）, length == MaxId
/// </summary>
public readonly byte[] RrpvSnapshot;
/// <summary>
/// Count of activated cache line handle
/// </summary>
public readonly int ActiveCount;
/// <summary>
/// Count of predictor ahead of current map position（aheadCount）
/// </summary>
public readonly int AheadCount;
/// <summary>
/// Max ID of the line being using or once used（maxId）
/// </summary>
public readonly int MaxId;
internal DebugSnapshot(
int activeCount,
int aheadCount,
int maxId,
byte[] rrpv
)
{
ActiveCount = activeCount;
AheadCount = aheadCount;
MaxId = maxId;
RrpvSnapshot = rrpv;
}
}
}
/**/
}
src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs
-66
View File
@@ -1,66 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
/// <summary>
///
/// </summary>
/// <param name="count">
///
/// </param>
/// <returns>
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(
int count = 1
)
{
count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count;
for (int i = 0; i < CacheLineCount; i++)
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i) ? 1 : 0;
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index
)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = SwissTable.Writer.TryRemove(_map, head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
return true;
}
return false;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs
-34
View File
@@ -1,34 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Exceptions"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace Nitload.Common.HighPerformance.Memory
{
#pragma warning disable CA1032 // Implement standard exception constructors
public sealed class L4PredictUnregisteredException : Exception
{
public L4PredictUnregisteredException(
int id
)
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
}
public sealed class L4PredictConflictException : Exception
{
public L4PredictConflictException(
nuint basePtr,
ushort policy,
int current
)
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
}
#pragma warning restore CA1032 // Implement standard exception constructors
}
src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs
-38
View File
@@ -1,38 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private IL4Predict CurrentPredict { get; set; }
public void ResetPredict(
IL4Predict predict
)
{
if (CurrentPredict is null)
{
CurrentPredict = predict;
return;
}
// TODO: write back all and register new predictor
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs
-146
View File
@@ -1,146 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predictor"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
[Flags]
public enum PredictResult
{
Success = 1,
Exhausted = 2,
}
public interface IL4Predict
{
PredictResult Predict(
Span<CacheLineDescription> node,
out int count
);
/// <summary>
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
/// name="cache" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Destination cache line buffer.
/// </param>
void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
);
/// <summary>
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Source cache line buffer.
/// </param>
void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
);
}
[StructLayout(LayoutKind.Explicit)]
public struct CacheLineDescription : IEquatable<CacheLineDescription>
{
public override readonly bool Equals(
object? obj
)
{
return (obj is CacheLineDescription prediction) &&
Address.Equals(prediction.Address) &&
(MapPolicy == prediction.MapPolicy);
}
public readonly bool Equals(
CacheLineDescription other
)
{
return
(Address == other.Address) && (MapPolicy == other.MapPolicy);
}
public override readonly int GetHashCode()
{
return (int)ComposePredictionKey64(Address, MapPolicy);
}
public readonly nuint GetHashCodeLong()
{
return ComposePredictionKey64(Address, MapPolicy);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint ComposePredictionKey64(
nuint address,
ushort mapPolicy
)
{
nuint x = address;
nuint y = mapPolicy;
unchecked
{
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
mixed ^= mixed >> 30;
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
mixed ^= mixed >> 27;
mixed *= (nuint)0x94d049bb133111ebUL;
mixed ^= mixed >> 31;
return mixed;
}
}
public static bool operator !=(
CacheLineDescription left,
CacheLineDescription right
)
{
return !(left == right);
}
public static bool operator ==(
CacheLineDescription left,
CacheLineDescription right
)
{
return left.Equals(right);
}
#pragma warning disable CA1051
[FieldOffset(0)]public nuint Address;
[FieldOffset(8)]public uint Size;
[FieldOffset(12)]public ushort MapPolicy;
[FieldOffset(14)]internal short Predictor;
#pragma warning restore CA1051
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs
-103
View File
@@ -1,103 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System.ComponentModel;
using System.Numerics;
namespace Nitload.Common.HighPerformance.Memory
{
public unsafe partial class L4
{
private partial void Prefetch()
{
if (CurrentPredict is null) {
return;
}
IL4Predict predict = CurrentPredict;
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
_exhausted = (result & PredictResult.Exhausted) != 0;
if (actualCount <= 0)
{
_exhausted = true;
return;
}
int missIndex = 0;
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!SwissTable.Reader.TryGet(_map, address, out _))
{
// miss and record
span[missIndex] = handle;
missIndex++;
}
}
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (SwissTable.Reader.TryGet(_map, address, out nuint handle))
{
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
_predictCount[index] = 0; // reset eviction counter
#region update predictor belonging
if (_predictCount[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref _predictCount[index]);
int mapCount = Volatile.Read(ref _mapCount[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref _mapCount[index], -decrement);
}
_predictCount[index]++;
#endregion
continue;
}
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = SwissTable.Writer.TrySet(_map, address, (nuint)head);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs
+217
View File
@@ -0,0 +1,217 @@
// -----------------------------------------------------------------------------
// file="L4.SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4
{
private const int BucketCapacity = 16;
private const uint CrcMagic = 0x7f1c9961;
private const int GroupCount = 1 << GroupCountLog;
private const int GroupCountLog = 10;
private const int IdealCount = GroupCount * BucketCapacity;//16384
private const int OffsetOfBucket = IdealCount;
private static readonly int TableBufferSize =
IdealCount + (GroupCount * Unsafe.SizeOf<SwissBucket>());
internal static readonly int BufferSize = TableBufferSize * 2;
private enum SwissControl : byte
{
Empty = 0b10000000,
Deleted = 0b10000001
}
[StructLayout(LayoutKind.Sequential)]
internal struct AddressSource
{
public ulong Address;
public ulong Description;
public override readonly bool Equals(
object obj
)
{
return obj is AddressSource source && this == source;
}
public override readonly int GetHashCode()
{
return HashCode.Combine(Address, Description);
}
public static bool operator !=(
AddressSource left,
AddressSource right
)
{
return !(left == right);
}
public static bool operator ==(
AddressSource left,
AddressSource right
)
{
return left.Address == right.Address && left.Description == right.Description;
}
}
[InlineArray(BucketCapacity)]
internal struct KeyArray
{
private AddressSource _element0;
}
internal struct SwissBucket
{
public KeyArray Keys;
public fixed ulong Values[BucketCapacity];
}
internal sealed unsafe class SwissTable : IDisposable
{
private byte* _buffer_read;
private byte* _buffer_write;
private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion);
private SwissTable(byte* buffers)
{
_buffer_read = buffers;
_buffer_write = buffers + TableBufferSize;
InitializeBuffer(_buffer_read);
InitializeBuffer(_buffer_write);
}
internal static SwissTable Create(byte* buffers)
{
ArgumentNullException.ThrowIfNull(buffers);
return new SwissTable(buffers);
}
/// <summary>
/// Reads the published buffer. Callers must prevent a swap while the read is in progress.
/// </summary>
public bool TryGet(AddressSource key, out ulong value)
{
_synchronization.EnterReadLock();
try
{
return SwissReaderX86.TryGet(_buffer_read, key, out value);
}
finally
{
_synchronization.ExitReadLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryAddOrSet(AddressSource key, ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryRemove(AddressSource key, out ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryRemove(_buffer_write, key, out value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced.
/// </summary>
public void PrepareWriteBuffer()
{
_synchronization.EnterWriteLock();
try
{
InitializeBuffer(_buffer_write);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Publishes the write buffer. The caller must provide the phase barrier around this operation.
/// </summary>
public void SwapBuffers()
{
_synchronization.EnterWriteLock();
try
{
byte* temporary = _buffer_read;
_buffer_read = _buffer_write;
_buffer_write = temporary;
}
finally
{
_synchronization.ExitWriteLock();
}
}
public void Dispose()
{
_synchronization.EnterWriteLock();
try
{
_buffer_read = null;
_buffer_write = null;
}
finally
{
_synchronization.ExitWriteLock();
}
_synchronization.Dispose();
}
private static void InitializeBuffer(byte* buffer)
{
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount);
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs
+200
View File
@@ -0,0 +1,200 @@
// -----------------------------------------------------------------------------
// file="L4.Swiss_X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private readonly unsafe struct SwissWriterX86
{
public static bool TryAddOrSet(
byte* buffer,
AddressSource key,
ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
uint first_tomb_group = 0;
int first_tomb_idx = -1;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte>* control_bytes_ptr = control_bytes + begin;
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr);
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
Vector128<byte> cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128);
int hash_mask = Sse2.MoveMask(cmp_hash);
int empty_mask = Sse2.MoveMask(cmp_empty);
int tomb_mask = Sse2.MoveMask(cmp_tomb);
if (tomb_mask != 0 && first_tomb_idx < 0)
{
int index = BitOperations.TrailingZeroCount(tomb_mask);
first_tomb_group = begin;
first_tomb_idx = index;
}
while (hash_mask != 0)
{
int index = BitOperations.TrailingZeroCount(hash_mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
bucket->Values[index] = value;
return true;
}
hash_mask &= ~(1 << index);
}
while (empty_mask != 0)
{
int index = BitOperations.TrailingZeroCount(empty_mask);
if (first_tomb_idx < 0)
{
SwissBucket* bucket = &buckets[begin];
bucket->Keys[index] = key;
bucket->Values[index] = value;
((byte*)control_bytes_ptr)[index] = target_hash;
} else
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
}
return true;
}
begin = (begin + 1) & (GroupCount - 1);
}
if (first_tomb_idx >= 0)
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
return true;
}
value = 0;
return false;
}
public static bool TryRemove(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted;
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
int empty_mask = Sse2.MoveMask(cmp_empty);
if (empty_mask != 0)
{
break;
}
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
private readonly unsafe struct SwissReaderX86
{
public static bool TryGet(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
if (Sse2.MoveMask(cmp_empty) != 0)
break;
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs
+9 -6
View File
@@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory
public partial class L4
{
private unsafe class TLSF
private unsafe struct TLSF
{
public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
private readonly short[] _bucket;
public static readonly nuint RequestedAllocation =
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
private readonly CacheLineMetadata* _spareMemory;
private readonly int _align;
private readonly int _maxLevel;
private int _metadataEnd;
private readonly Ptr64 _basePtr;
private fixed short _bucket[64];
private readonly uint _maxAllocSize;
private readonly uint _totalSize;
@@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2];
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0;
@@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory
idx = candidate->NextLevel;
}
}
for (int i = level + 1; i < _bucket.Length; i++)
for (int i = level + 1; i < _bucket[63]; i++)
{
int idx = _bucket[i];
if (idx != 0)
src/Nitload.Common/HighPerformance/Memory/L4.cs
+26 -133
View File
@@ -5,49 +5,20 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4 : IDisposable
{
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
/// <summary>
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
/// </summary>
private readonly CacheLineMetadata* _head;
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
private int _aheadCount;
private readonly int _bindedNuma;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SwissTable _map;
private readonly TLSF _bufferAllocator;
private SwissTable _addressMap;
private TLSF _bufferAllocator;
public L4(
int bindedNuma,
@@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
ByteCapacity = normalizedBufferSize;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
AllocSize = swissBufferOffset + (nuint)BufferSize;
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
_buffer += normalizedBufferSize;
_head = (CacheLineMetadata*)ptr;
ptr += headSize;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
for (uint i = 1; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(0);
}
_addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
}
public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design",
"CA1063:Implement IDisposable correctly",
Justification = """
Only owns native memory via Numa.Free();
no managed IDisposable fields.
Dispose(bool) pattern is unnecessary.
""")]
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_map.Dispose();
#pragma warning disable CA1816
GC.SuppressFinalize(_bufferAllocator);
#pragma warning restore CA1816
// init order is fallow by align requirement of each component.
_addressMap.Dispose();
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(
ref CacheLineDescription prediction
)
internal void Collect()
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{
try
{
if (_aheadCount < PredictionLeastAhead)
{
// do prefetch in work load now, sync
Prefetch();
}
}
finally
{
_prefetching = 0;
}
}
}
while (true)
{
if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
}
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Collect();
}
private partial void Prefetch();
internal void Free(
int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Free(handle);
}
internal bool TryAllocate(
uint size,
out int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _bufferAllocator.TryAlloc(size, out handle);
}
}
}