youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

after merge

Browse Source
This commit is contained in:
youmuliu
2026-10-12 04:14:21 +08:00
parent 2bef14493f 0cef5aa517
commit 76a9dfa5af
15 changed files with 730 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs
+14 -5
View File
@@ -27,6 +27,8 @@ namespace Nitload.Common.HighPerformance.Memory
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(16)] internal AllocationState State;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
@@ -43,20 +45,27 @@ namespace Nitload.Common.HighPerformance.Memory
[FieldOffset(8)] public uint PositionMask;
[FieldOffset(12)] public uint SizeMask;
#endregion
[FieldOffset(16)] internal AllocationState State;
#endregion
}
#pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit, Size = 16)]
public struct Handle : IDisposable
public unsafe struct Handle : IDisposable
{
[FieldOffset(0)] private nuint _handle;
[FieldOffset(8)] private nuint _reserved;
internal Handle(
nuint handle,
nuint reserved
)
{
_handle = handle;
_reserved = reserved;
}
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
public Handle() { }
src/Nitload.Common/HighPerformance/Memory/L4.Map.cs
+98
View File
@@ -0,0 +1,98 @@
// -----------------------------------------------------------------------------
// file="L4.Map"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public unsafe partial class L4
{
private const int _low_predict = 512;
private int _predict_count;
private int _swiss_reader_counter;
private int _swiss_swap_lock;
private int _swiss_writer_lock;
public partial Handle Map(
AddressSource source
)
{
ulong handle;
if (Volatile.Read(ref _predict_count) < _low_predict)
{
if (Interlocked.CompareExchange(ref _swiss_writer_lock, 1, 0) == 0)
{
try
{
Predict();
_ = Interlocked.Exchange(ref _swiss_swap_lock, 1);
while (Volatile.Read(ref _swiss_reader_counter) != 0) { }
_addressMap.SwapBuffers();
if (_addressMap.TryGet(source, out handle)) {
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
}
}
finally
{
_ = Interlocked.Exchange(ref _swiss_swap_lock, 0);
_ = Interlocked.Exchange(ref _swiss_writer_lock, 0);
}
} else if (!TryGetAddress(source, out handle)) {
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
}
} else if (!TryGetAddress(source, out handle)) {
throw new InvalidOperationException($"Unexpected map request: [address source: 0x{source.Address:x16}; layout profile: {source.Description:x16}].");
}
CacheLineMetadata* metadata = (CacheLineMetadata*)handle;
// process metadata, unfinished
uint offset = metadata->PositionMask;
nuint address = /*base ptr + */ offset;
_ = Interlocked.Decrement(ref _predict_count);
return new Handle(address, (nuint)handle);
}
private bool TryGetAddress(
AddressSource source,
out ulong handle
)
{
while (true)
{
while (Volatile.Read(ref _swiss_swap_lock) != 0) { }
_ = Interlocked.Increment(ref _swiss_reader_counter);
if (Volatile.Read(ref _swiss_swap_lock) == 0)
{
break;
}
_ = Interlocked.Decrement(ref _swiss_reader_counter);
}
try
{
return _addressMap.TryGet(source, out handle);
}
finally
{
_ = Interlocked.Decrement(ref _swiss_reader_counter);
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs
+22
View File
@@ -0,0 +1,22 @@
// -----------------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public unsafe partial class L4
{
internal partial void Predict() { }
}
}
src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs
+93 -113
View File
@@ -8,7 +8,7 @@
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.HighPerformance.Memory
{
@@ -16,8 +16,6 @@ namespace Nitload.Common.HighPerformance.Memory
{
private const int BucketCapacity = 16;
private const uint CrcMagic = 0x7f1c9961;
private const int GroupCount = 1 << GroupCountLog;
private const int GroupCountLog = 10;
private const int IdealCount = GroupCount * BucketCapacity;//16384
@@ -36,17 +34,24 @@ namespace Nitload.Common.HighPerformance.Memory
}
[StructLayout(LayoutKind.Sequential)]
internal struct AddressSource
public struct AddressSource : IEquatable<AddressSource>
{
public ulong Address;
public ulong Description;
public override readonly bool Equals(
object obj
public readonly bool Equals(
AddressSource other
)
{
return obj is AddressSource source && this == source;
return this == other;
}
public override readonly bool Equals(
object? obj
)
{
return obj is AddressSource source && Equals(source);
}
public override readonly int GetHashCode()
@@ -88,127 +93,102 @@ namespace Nitload.Common.HighPerformance.Memory
}
internal sealed unsafe class SwissTable : IDisposable
internal unsafe struct SwissTable : IDisposable
{
private byte* _buffer_read;
private byte* _buffer_write;
private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion);
private SwissTable(byte* buffers)
private ulong _published_buffer;
private SwissTable(
byte* buffers
)
{
_buffer_read = buffers;
_published_buffer = (ulong)buffers;
_buffer_write = buffers + TableBufferSize;
InitializeBuffer(_buffer_read);
InitializeBuffer(buffers);
InitializeBuffer(_buffer_write);
}
internal static SwissTable Create(byte* buffers)
internal byte* ReadBufferForLockFreeBenchmark => (byte*)Volatile.Read(ref _published_buffer);
internal readonly byte* WriteBufferForLockFreeBenchmark => _buffer_write;
public void Dispose()
{
_ = Interlocked.Exchange(ref _published_buffer, 0);
_buffer_write = null;
}
/// <summary>
/// Publishes the write buffer. The caller must provide the phase barrier around this
/// operation.
/// </summary>
public void SwapBuffers()
{
ulong previousBuffer = Interlocked.Exchange(ref _published_buffer, (ulong)_buffer_write);
_buffer_write = (byte*)previousBuffer;
}
/// <summary>
/// Writes to the inactive buffer. The caller must serialize all writes.
/// </summary>
public readonly bool TryAddOrSet(
AddressSource key,
ulong value
)
{
if (Sse42.IsSupported) {
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
}
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
}
/// <summary>
/// Reads the published buffer. The caller must prevent swaps until all reads finish.
/// </summary>
public bool TryGet(
AddressSource key,
out ulong value
)
{
byte* publishedBuffer = (byte*)Volatile.Read(ref _published_buffer);
if (Sse42.IsSupported) {
return SwissReaderX86.TryGet(publishedBuffer, key, out value);
}
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
}
/// <summary>
/// Removes from the inactive buffer. The caller must serialize all writes.
/// </summary>
public bool TryRemove(
AddressSource key,
out ulong value
)
{
if (Sse42.IsSupported) {
return SwissWriterX86.TryRemove(_buffer_write, key, out value);
}
throw new NotImplementedException("The Swiss backend is not implemented for this ISA.");
}
internal static SwissTable Create(
byte* buffers
)
{
ArgumentNullException.ThrowIfNull(buffers);
return new SwissTable(buffers);
}
/// <summary>
/// Reads the published buffer. Callers must prevent a swap while the read is in progress.
/// </summary>
public bool TryGet(AddressSource key, out ulong value)
private static void InitializeBuffer(
byte* buffer
)
{
_synchronization.EnterReadLock();
try
{
return SwissReaderX86.TryGet(_buffer_read, key, out value);
}
finally
{
_synchronization.ExitReadLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryAddOrSet(AddressSource key, ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryRemove(AddressSource key, out ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryRemove(_buffer_write, key, out value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced.
/// </summary>
public void PrepareWriteBuffer()
{
_synchronization.EnterWriteLock();
try
{
InitializeBuffer(_buffer_write);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Publishes the write buffer. The caller must provide the phase barrier around this operation.
/// </summary>
public void SwapBuffers()
{
_synchronization.EnterWriteLock();
try
{
byte* temporary = _buffer_read;
_buffer_read = _buffer_write;
_buffer_write = temporary;
}
finally
{
_synchronization.ExitWriteLock();
}
}
public void Dispose()
{
_synchronization.EnterWriteLock();
try
{
_buffer_read = null;
_buffer_write = null;
}
finally
{
_synchronization.ExitWriteLock();
}
_synchronization.Dispose();
}
private static void InitializeBuffer(byte* buffer)
{
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount);
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, IdealCount);
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs
+26 -5
View File
@@ -6,6 +6,7 @@
// -----------------------------------------------------------------------------
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
@@ -14,7 +15,25 @@ namespace Nitload.Common.HighPerformance.Memory
public partial class L4
{
private readonly unsafe struct SwissWriterX86
private const uint CrcMagic = 0x7f1c9961;
internal static class SwissHashX64
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static uint Compute(
AddressSource key
)
{
ulong hash = CrcMagic;
hash = Sse42.X64.Crc32(hash, key.Address);
hash = Sse42.X64.Crc32(hash, key.Description);
return (uint)hash;
}
}
internal readonly unsafe struct SwissWriterX86
{
public static bool TryAddOrSet(
@@ -26,7 +45,7 @@ namespace Nitload.Common.HighPerformance.Memory
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint hash = SwissHashX64.Compute(key);
uint begin = hash % GroupCount;
uint first_tomb_group = 0;
@@ -108,7 +127,7 @@ namespace Nitload.Common.HighPerformance.Memory
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint hash = SwissHashX64.Compute(key);
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
@@ -149,7 +168,7 @@ namespace Nitload.Common.HighPerformance.Memory
}
private readonly unsafe struct SwissReaderX86
internal readonly unsafe struct SwissReaderX86
{
public static bool TryGet(
@@ -161,7 +180,7 @@ namespace Nitload.Common.HighPerformance.Memory
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint hash = SwissHashX64.Compute(key);
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
@@ -186,7 +205,9 @@ namespace Nitload.Common.HighPerformance.Memory
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
if (Sse2.MoveMask(cmp_empty) != 0)
{
break;
}
begin = (begin + 1) & (GroupCount - 1);
}
src/Nitload.Common/HighPerformance/Memory/L4.cs
+6
View File
@@ -54,6 +54,10 @@ namespace Nitload.Common.HighPerformance.Memory
GC.SuppressFinalize(this);
}
public partial Handle Map(
AddressSource source
);
internal void Collect()
{
ObjectDisposedException.ThrowIf(_disposed, this);
@@ -77,5 +81,7 @@ namespace Nitload.Common.HighPerformance.Memory
return _bufferAllocator.TryAlloc(size, out handle);
}
internal partial void Predict();
}
}