youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

swiss x86 is briefly ok, ready for test

Browse Source
This commit is contained in:
Youmu Liu
2026-10-09 18:17:03 +08:00
parent 2e7c3040f4
commit 3a2b123478
15 changed files with 457 additions and 1317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Benchmark/SwissBenchmark.cs
-485
View File
@@ -1,485 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissBenchmark"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using BenchmarkDotNet.Attributes;
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace Nitload.Benchmark
{
/// <summary>
/// Benchmark comparing SwissTable against <see cref="Dictionary{TKey, TValue}" /> under
/// realistic L4 software cache address-mapping workloads. <para>/// L4 scenario
/// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned)
/// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) ///
/// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// -
/// Working set: 512–4096 active cache line mappings /// </para>
/// </summary>
[MemoryDiagnoser]
public class SwissTableBench
{
/// <summary>
/// L4 cache line size in bytes (per L4_design.md).
/// </summary>
private const int CacheLineSize = 1024;
/// <summary>
/// Simulated heap base address for pointer-like keys.
/// </summary>
private const long HeapBase = 0x7FFF_0000_0000L;
/// <summary>
/// Number of lookup operations per Lookup benchmark call.
/// </summary>
private const int LookupIterations = 8192;
/// <summary>
/// Fraction of steady-state ops that are cache misses.
/// </summary>
private const double MissRate = 0.05;
/// <summary>
/// Number of steady-state operations per call. Each op is either a lookup (hit) or an
/// evict+insert+lookup (miss).
/// </summary>
private const int SteadyStateOps = 4096;
/// <summary>
/// Indices into _residentKeys for eviction targets (cold entries, tail region).
/// </summary>
private int[] _evictOrder = null!;
/// <summary>
/// Additional keys not in the table, for triggering misses.
/// </summary>
private nuint[] _externalKeys = null!;
// ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ──
/// <summary>
/// Keys for initial table fill (simulates cached addresses).
/// </summary>
private nuint[] _residentKeys = null!;
/// <summary>
/// Pre-built steady-state operation script. Each entry: positive index → lookup
/// _residentKeys[idx], negative value → cache miss trigger (evict + insert from
/// _externalKeys).
/// </summary>
private int[] _steadyStateScript = null!;
/// <summary>
/// Uniform random lookup sequence (baseline comparison).
/// </summary>
private nuint[] _uniformLookups = null!;
/// <summary>
/// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time).
/// </summary>
private nuint[] _zipfianLookups = null!;
private Dictionary<nuint, nuint> _dictLookup = null!;
private Dictionary<nuint, nuint> _freshDict = null!;
// ── Fresh instances for insert/mixed (rebuilt each iteration) ──
private SwissTable _freshSwiss = null!;
// ── Pre-filled lookup targets ──
private SwissTable _swissLookup = null!;
/// <summary>
/// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB /
/// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity.
/// </summary>
[Params(512, 1024, 4096)]
public int KeyCount;
[GlobalCleanup]
public void Cleanup()
{
_dictLookup.Clear();
_dictLookup.TrimExcess();
}
// ════════════════════════════════════════════════════════════
// Insert — bulk fill with scattered pointer addresses
// Simulates L4 initial cache warm-up phase.
// ════════════════════════════════════════════════════════════
[Benchmark]
public void Dictionary_Insert()
{
Dictionary<nuint, nuint> dict = _freshDict;
nuint[] keys = _residentKeys;
for (int i = 0; i < keys.Length; i++) {
dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF);
}
}
// ════════════════════════════════════════════════════════════
// Lookup — Uniform (baseline comparison)
// ════════════════════════════════════════════════════════════
[Benchmark]
public nuint Dictionary_Lookup_Uniform()
{
nuint sum = 0;
nuint[] keys = _uniformLookups;
Dictionary<nuint, nuint> dict = _dictLookup;
for (int i = 0; i < keys.Length; i++)
{
if (dict.TryGetValue(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// Lookup — Zipfian (L4 hot path, skewed access)
// 20% of cache lines receive 80% of accesses.
// This is the most representative L4 benchmark.
// ════════════════════════════════════════════════════════════
[Benchmark(Baseline = true)]
public nuint Dictionary_Lookup_Zipfian()
{
nuint sum = 0;
nuint[] keys = _zipfianLookups;
Dictionary<nuint, nuint> dict = _dictLookup;
for (int i = 0; i < keys.Length; i++)
{
if (dict.TryGetValue(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// SteadyState — the KEY L4 benchmark
// Pre-filled table at capacity, then 4096 operations:
// ~95% are lookups (cache hits)
// ~5% are miss → evict cold entry → insert new entry → lookup new
// This directly simulates L4's main runtime loop.
// ════════════════════════════════════════════════════════════
[Benchmark]
public nuint Dictionary_SteadyState()
{
nuint sum = 0;
Dictionary<nuint, nuint> dict = _freshDict;
nuint[] resident = _residentKeys;
nuint[] external = _externalKeys;
int[] script = _steadyStateScript;
int[] evict = _evictOrder;
int evictCursor = 0;
int externalCursor = 0;
for (int i = 0; i < script.Length; i++)
{
int op = script[i];
if (op >= 0)
{
// Cache hit: lookup existing entry
if (dict.TryGetValue(resident[op], out nuint value))
{
sum += value;
}
} else
{
// Cache miss: evict a cold entry, insert a new one, then lookup
int evictIdx = evict[evictCursor % evict.Length];
evictCursor++;
dict.Remove(resident[evictIdx]);
int extIdx = externalCursor % external.Length;
externalCursor++;
nuint newKey = external[extIdx];
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
dict[newKey] = newVal;
if (dict.TryGetValue(newKey, out nuint v)) {
sum += v;
}
}
}
return sum;
}
[IterationSetup(Targets = [
nameof(SwissTable_Insert),
nameof(SwissTable_SteadyState),
nameof(Dictionary_Insert),
nameof(Dictionary_SteadyState)
])]
public void IterationSetup()
{
_freshSwiss = SwissTable.Create();
_freshDict = new Dictionary<nuint, nuint>(KeyCount);
// Pre-fill for steady-state benchmarks
for (int i = 0; i < KeyCount; i++)
{
nuint key = _residentKeys[i];
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF);
SwissTable.Writer.TrySet(_freshSwiss, key, val);
_freshDict[key] = val;
}
}
// ════════════════════════════════════════════════════════════
// Setup / Teardown
// ════════════════════════════════════════════════════════════
[GlobalSetup]
public void Setup()
{
Random rng = new Random(42);
// ── Generate pointer-like keys (HeapBase + random_offset * 1024) ──
_residentKeys = GeneratePointerKeys(KeyCount, rng);
_externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation
// ── Build lookup sequences ──
_zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng);
_uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng);
// ── Build steady-state operation script ──
_steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng);
// ── Build eviction order (cold tail entries first) ──
_evictOrder = new int[KeyCount];
for (int i = 0; i < KeyCount; i++) {
_evictOrder[i] = i;
}
// Shuffle so eviction isn't just sequential
Shuffle(_evictOrder, rng);
// ── Pre-fill lookup tables ──
_swissLookup = SwissTable.Create();
_dictLookup = new Dictionary<nuint, nuint>(KeyCount);
for (int i = 0; i < KeyCount; i++)
{
nuint key = _residentKeys[i];
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer
SwissTable.Writer.TrySet(_swissLookup, key, val);
_dictLookup[key] = val;
}
}
[Benchmark]
public void SwissTable_Insert()
{
SwissTable swiss = _freshSwiss;
nuint[] keys = _residentKeys;
for (int i = 0; i < keys.Length; i++) {
SwissTable.Writer.TrySet(swiss, keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF));
}
}
[Benchmark]
public nuint SwissTable_Lookup_Uniform()
{
nuint sum = 0;
nuint[] keys = _uniformLookups;
SwissTable swiss = _swissLookup;
for (int i = 0; i < keys.Length; i++)
{
if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
[Benchmark]
public nuint SwissTable_Lookup_Zipfian()
{
nuint sum = 0;
nuint[] keys = _zipfianLookups;
SwissTable swiss = _swissLookup;
for (int i = 0; i < keys.Length; i++)
{
if (SwissTable.Reader.TryGet(swiss, keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
[Benchmark]
public nuint SwissTable_SteadyState()
{
nuint sum = 0;
SwissTable swiss = _freshSwiss;
nuint[] resident = _residentKeys;
nuint[] external = _externalKeys;
int[] script = _steadyStateScript;
int[] evict = _evictOrder;
int evictCursor = 0;
int externalCursor = 0;
for (int i = 0; i < script.Length; i++)
{
int op = script[i];
if (op >= 0)
{
// Cache hit: lookup existing entry
if (SwissTable.Reader.TryGet(swiss, resident[op], out nuint value))
{
sum += value;
}
} else
{
// Cache miss: evict a cold entry, insert a new one, then lookup
int evictIdx = evict[evictCursor % evict.Length];
evictCursor++;
SwissTable.Writer.TryRemove(swiss, resident[evictIdx], out _);
int extIdx = externalCursor % external.Length;
externalCursor++;
nuint newKey = external[extIdx];
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
SwissTable.Writer.TryAdd(swiss, newKey, newVal);
if (SwissTable.Reader.TryGet(swiss, newKey, out nuint v)) {
sum += v;
}
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// Data generation helpers
// ════════════════════════════════════════════════════════════
/// <summary>
/// Generate <paramref name="count" /> unique pointer-like keys. Each key = HeapBase +
/// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses
/// scattered across a ~1 GB virtual address range.
/// </summary>
private static nuint[] GeneratePointerKeys(
int count,
Random rng
)
{
nuint[] keys = new nuint[count];
HashSet<long> used = new HashSet<long>(count);
for (int i = 0; i < count; i++)
{
long offset;
do
{
// Random offset within ~1M cache lines (1 GB range)
offset = (long)rng.Next(0, 1 << 20) * CacheLineSize;
} while (!used.Add(offset));
keys[i] = (nuint)(HeapBase + offset);
}
return keys;
}
/// <summary>
/// Generate a steady-state operation script. Each entry: non-negative value = index into
/// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert →
/// lookup). Lookup indices are Zipfian-skewed.
/// </summary>
private static int[] GenerateSteadyStateScript(
int keyCount,
int opCount,
double missRate,
Random rng
)
{
int hotSetSize = Math.Max(1, keyCount / 5);
int[] script = new int[opCount];
for (int i = 0; i < opCount; i++)
{
if (rng.NextDouble() < missRate)
{
script[i] = -1; // miss
} else
{
// Zipfian hit
if (rng.NextDouble() < 0.8)
{
script[i] = rng.Next(0, hotSetSize);
} else
{
script[i] = rng.Next(hotSetSize, keyCount);
}
}
}
return script;
}
/// <summary>
/// Generate uniform random lookup sequence (baseline for comparison with Zipfian).
/// </summary>
private static nuint[] GenerateUniformLookups(
nuint[] sourceKeys,
int count,
Random rng
)
{
nuint[] lookups = new nuint[count];
for (int i = 0; i < count; i++) {
lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)];
}
return lookups;
}
/// <summary>
/// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys
/// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This
/// models matrix computation where a small panel working set is reused heavily.
/// </summary>
private static nuint[] GenerateZipfianLookups(
nuint[] sourceKeys,
int count,
Random rng
)
{
int hotSetSize = Math.Max(1, sourceKeys.Length / 5);
nuint[] lookups = new nuint[count];
for (int i = 0; i < count; i++)
{
if (rng.NextDouble() < 0.8)
{
lookups[i] = sourceKeys[rng.Next(0, hotSetSize)];
} else
{
lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)];
}
}
return lookups;
}
/// <summary>
/// Fisher-Yates shuffle.
/// </summary>
private static void Shuffle(
int[] array,
Random rng
)
{
for (int i = array.Length - 1; i > 0; i--)
{
int j = rng.Next(0, i + 1);
(array[i], array[j]) = (array[j], array[i]);
}
}
}
}
src/Nitload.Common/Collections/SwissTable.cs
-125
View File
@@ -1,125 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Nitload.Common.Collections
{
public sealed unsafe partial class SwissTable
{
private const int BucketSize = 16;
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
private const int GroupCount = 1 << EntryMaskWidth;
public const int MaxCapacity = GroupCount * BucketSize; // 256 entries per bucket, 1024 buckets
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<byte>() + (nuint)Unsafe.SizeOf<M128>();
private byte* _controlBytes;
private SwissTable(
byte* buffer
)
{
_controlBytes = buffer;
}
private M128* KvpArray => (M128*)(_controlBytes + MaxCapacity);
private static void Initialize(
byte* buffer
)
{
Unsafe.InitBlock(buffer, 0b10000000, BucketSize);
}
private unsafe struct Bucket
{
public BucketControl Control;
public BucketKeyStorage Keys;
public BucketValueStorage Values;
}
public static class Reader
{
public static bool TryGet(
SwissTable table,
nuint key,
out nuint value
)
{
throw new NotImplementedException("SwissTable reader is awaiting a redesigned storage and lookup implementation.");
}
}
public static class Writer
{
public static bool TryAdd(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and insertion implementation.");
}
public static bool TryRemove(
SwissTable table,
nuint key,
out nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and removal implementation.");
}
public static bool TrySet(
SwissTable table,
nuint key,
nuint value
)
{
throw new NotImplementedException("SwissTable writer is awaiting a redesigned storage and update implementation.");
}
}
[InlineArray(BucketSize)]
private struct BucketControl
{
public byte Control;
}
[InlineArray(BucketSize)]
private struct BucketKeyStorage
{
public nuint value;
}
[InlineArray(BucketSize)]
private struct BucketValueStorage
{
public nuint value;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs
+5 -25
View File
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: MIT // SPDX-License-Identifier: MIT
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory namespace Nitload.Common.HighPerformance.Memory
@@ -23,15 +22,11 @@ namespace Nitload.Common.HighPerformance.Memory
} }
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)] [StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata private struct CacheLineMetadata
{ {
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U; internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(24)] internal nuint Origin;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>(); public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list #region tlsf linked list
@@ -50,32 +45,17 @@ namespace Nitload.Common.HighPerformance.Memory
#endregion #endregion
#region mask
[FieldOffset(16)] internal AllocationState State; [FieldOffset(16)] internal AllocationState State;
[FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion
} }
#pragma warning disable IDE0032 #pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit)] [StructLayout(LayoutKind.Explicit, Size = 16)]
public unsafe struct Handle : IDisposable public struct Handle : IDisposable
{ {
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
[FieldOffset(0)] private nuint _handle; [FieldOffset(0)] private nuint _handle;
[FieldOffset(8)] private nuint _reserved;
private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{
_handle = handle;
_head = cacheRefCount;
}
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
public Handle() { } public Handle() { }
@@ -88,7 +68,7 @@ namespace Nitload.Common.HighPerformance.Memory
return; return;
} }
_handle = 0; _handle = 0;
_ = Interlocked.Decrement(ref _head->RefCount); _reserved = 0;
} }
} }
src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs
-93
View File
@@ -1,93 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Debug"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
/**/
public partial class L4
{
public void DebugAgeAll()
{
// AgeAll();
}
public bool DebugContains(
nuint origin
)
{
return SwissTable.Reader.TryGet(_map, origin, out _);
}
public int DebugEvict(
int count = 1
)
{
return Evict(count);
}
public unsafe DebugSnapshot GetDebugSnapshot()
{
throw new NotImplementedException();
/*
int maxId = _maxId;
byte[] rrpv = new byte[maxId];
byte* p = (byte*)_eviction;
for (int i = 0; i < maxId; i++) {
rrpv[i] = p[i];
}
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
/**/
}
public readonly struct DebugSnapshot
{
/// <summary>
/// RRPV value for each line（0~3）, length == MaxId
/// </summary>
public readonly byte[] RrpvSnapshot;
/// <summary>
/// Count of activated cache line handle
/// </summary>
public readonly int ActiveCount;
/// <summary>
/// Count of predictor ahead of current map position（aheadCount）
/// </summary>
public readonly int AheadCount;
/// <summary>
/// Max ID of the line being using or once used（maxId）
/// </summary>
public readonly int MaxId;
internal DebugSnapshot(
int activeCount,
int aheadCount,
int maxId,
byte[] rrpv
)
{
ActiveCount = activeCount;
AheadCount = aheadCount;
MaxId = maxId;
RrpvSnapshot = rrpv;
}
}
}
/**/
}
src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs
-66
View File
@@ -1,66 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
/// <summary>
///
/// </summary>
/// <param name="count">
///
/// </param>
/// <returns>
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(
int count = 1
)
{
count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count;
for (int i = 0; i < CacheLineCount; i++)
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i) ? 1 : 0;
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index
)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = SwissTable.Writer.TryRemove(_map, head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
return true;
}
return false;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs
-34
View File
@@ -1,34 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Exceptions"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace Nitload.Common.HighPerformance.Memory
{
#pragma warning disable CA1032 // Implement standard exception constructors
public sealed class L4PredictUnregisteredException : Exception
{
public L4PredictUnregisteredException(
int id
)
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
}
public sealed class L4PredictConflictException : Exception
{
public L4PredictConflictException(
nuint basePtr,
ushort policy,
int current
)
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
}
#pragma warning restore CA1032 // Implement standard exception constructors
}
src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs
-38
View File
@@ -1,38 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private IL4Predict CurrentPredict { get; set; }
public void ResetPredict(
IL4Predict predict
)
{
if (CurrentPredict is null)
{
CurrentPredict = predict;
return;
}
// TODO: write back all and register new predictor
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs
-146
View File
@@ -1,146 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predictor"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
[Flags]
public enum PredictResult
{
Success = 1,
Exhausted = 2,
}
public interface IL4Predict
{
PredictResult Predict(
Span<CacheLineDescription> node,
out int count
);
/// <summary>
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
/// name="cache" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Destination cache line buffer.
/// </param>
void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
);
/// <summary>
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Source cache line buffer.
/// </param>
void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
);
}
[StructLayout(LayoutKind.Explicit)]
public struct CacheLineDescription : IEquatable<CacheLineDescription>
{
public override readonly bool Equals(
object? obj
)
{
return (obj is CacheLineDescription prediction) &&
Address.Equals(prediction.Address) &&
(MapPolicy == prediction.MapPolicy);
}
public readonly bool Equals(
CacheLineDescription other
)
{
return
(Address == other.Address) && (MapPolicy == other.MapPolicy);
}
public override readonly int GetHashCode()
{
return (int)ComposePredictionKey64(Address, MapPolicy);
}
public readonly nuint GetHashCodeLong()
{
return ComposePredictionKey64(Address, MapPolicy);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint ComposePredictionKey64(
nuint address,
ushort mapPolicy
)
{
nuint x = address;
nuint y = mapPolicy;
unchecked
{
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
mixed ^= mixed >> 30;
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
mixed ^= mixed >> 27;
mixed *= (nuint)0x94d049bb133111ebUL;
mixed ^= mixed >> 31;
return mixed;
}
}
public static bool operator !=(
CacheLineDescription left,
CacheLineDescription right
)
{
return !(left == right);
}
public static bool operator ==(
CacheLineDescription left,
CacheLineDescription right
)
{
return left.Equals(right);
}
#pragma warning disable CA1051
[FieldOffset(0)]public nuint Address;
[FieldOffset(8)]public uint Size;
[FieldOffset(12)]public ushort MapPolicy;
[FieldOffset(14)]internal short Predictor;
#pragma warning restore CA1051
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs
-103
View File
@@ -1,103 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System.ComponentModel;
using System.Numerics;
namespace Nitload.Common.HighPerformance.Memory
{
public unsafe partial class L4
{
private partial void Prefetch()
{
if (CurrentPredict is null) {
return;
}
IL4Predict predict = CurrentPredict;
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
_exhausted = (result & PredictResult.Exhausted) != 0;
if (actualCount <= 0)
{
_exhausted = true;
return;
}
int missIndex = 0;
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!SwissTable.Reader.TryGet(_map, address, out _))
{
// miss and record
span[missIndex] = handle;
missIndex++;
}
}
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (SwissTable.Reader.TryGet(_map, address, out nuint handle))
{
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
_predictCount[index] = 0; // reset eviction counter
#region update predictor belonging
if (_predictCount[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref _predictCount[index]);
int mapCount = Volatile.Read(ref _mapCount[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref _mapCount[index], -decrement);
}
_predictCount[index]++;
#endregion
continue;
}
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = SwissTable.Writer.TrySet(_map, address, (nuint)head);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs
+217
View File
@@ -0,0 +1,217 @@
// -----------------------------------------------------------------------------
// file="L4.SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4
{
private const int BucketCapacity = 16;
private const uint CrcMagic = 0x7f1c9961;
private const int GroupCount = 1 << GroupCountLog;
private const int GroupCountLog = 10;
private const int IdealCount = GroupCount * BucketCapacity;//16384
private const int OffsetOfBucket = IdealCount;
private static readonly int TableBufferSize =
IdealCount + (GroupCount * Unsafe.SizeOf<SwissBucket>());
internal static readonly int BufferSize = TableBufferSize * 2;
private enum SwissControl : byte
{
Empty = 0b10000000,
Deleted = 0b10000001
}
[StructLayout(LayoutKind.Sequential)]
internal struct AddressSource
{
public ulong Address;
public ulong Description;
public override readonly bool Equals(
object obj
)
{
return obj is AddressSource source && this == source;
}
public override readonly int GetHashCode()
{
return HashCode.Combine(Address, Description);
}
public static bool operator !=(
AddressSource left,
AddressSource right
)
{
return !(left == right);
}
public static bool operator ==(
AddressSource left,
AddressSource right
)
{
return left.Address == right.Address && left.Description == right.Description;
}
}
[InlineArray(BucketCapacity)]
internal struct KeyArray
{
private AddressSource _element0;
}
internal struct SwissBucket
{
public KeyArray Keys;
public fixed ulong Values[BucketCapacity];
}
internal sealed unsafe class SwissTable : IDisposable
{
private byte* _buffer_read;
private byte* _buffer_write;
private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion);
private SwissTable(byte* buffers)
{
_buffer_read = buffers;
_buffer_write = buffers + TableBufferSize;
InitializeBuffer(_buffer_read);
InitializeBuffer(_buffer_write);
}
internal static SwissTable Create(byte* buffers)
{
ArgumentNullException.ThrowIfNull(buffers);
return new SwissTable(buffers);
}
/// <summary>
/// Reads the published buffer. Callers must prevent a swap while the read is in progress.
/// </summary>
public bool TryGet(AddressSource key, out ulong value)
{
_synchronization.EnterReadLock();
try
{
return SwissReaderX86.TryGet(_buffer_read, key, out value);
}
finally
{
_synchronization.ExitReadLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryAddOrSet(AddressSource key, ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryRemove(AddressSource key, out ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryRemove(_buffer_write, key, out value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced.
/// </summary>
public void PrepareWriteBuffer()
{
_synchronization.EnterWriteLock();
try
{
InitializeBuffer(_buffer_write);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Publishes the write buffer. The caller must provide the phase barrier around this operation.
/// </summary>
public void SwapBuffers()
{
_synchronization.EnterWriteLock();
try
{
byte* temporary = _buffer_read;
_buffer_read = _buffer_write;
_buffer_write = temporary;
}
finally
{
_synchronization.ExitWriteLock();
}
}
public void Dispose()
{
_synchronization.EnterWriteLock();
try
{
_buffer_read = null;
_buffer_write = null;
}
finally
{
_synchronization.ExitWriteLock();
}
_synchronization.Dispose();
}
private static void InitializeBuffer(byte* buffer)
{
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount);
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs
+200
View File
@@ -0,0 +1,200 @@
// -----------------------------------------------------------------------------
// file="L4.Swiss_X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private readonly unsafe struct SwissWriterX86
{
public static bool TryAddOrSet(
byte* buffer,
AddressSource key,
ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
uint first_tomb_group = 0;
int first_tomb_idx = -1;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte>* control_bytes_ptr = control_bytes + begin;
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr);
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
Vector128<byte> cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128);
int hash_mask = Sse2.MoveMask(cmp_hash);
int empty_mask = Sse2.MoveMask(cmp_empty);
int tomb_mask = Sse2.MoveMask(cmp_tomb);
if (tomb_mask != 0 && first_tomb_idx < 0)
{
int index = BitOperations.TrailingZeroCount(tomb_mask);
first_tomb_group = begin;
first_tomb_idx = index;
}
while (hash_mask != 0)
{
int index = BitOperations.TrailingZeroCount(hash_mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
bucket->Values[index] = value;
return true;
}
hash_mask &= ~(1 << index);
}
while (empty_mask != 0)
{
int index = BitOperations.TrailingZeroCount(empty_mask);
if (first_tomb_idx < 0)
{
SwissBucket* bucket = &buckets[begin];
bucket->Keys[index] = key;
bucket->Values[index] = value;
((byte*)control_bytes_ptr)[index] = target_hash;
} else
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
}
return true;
}
begin = (begin + 1) & (GroupCount - 1);
}
if (first_tomb_idx >= 0)
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
return true;
}
value = 0;
return false;
}
public static bool TryRemove(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted;
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
int empty_mask = Sse2.MoveMask(cmp_empty);
if (empty_mask != 0)
{
break;
}
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
private readonly unsafe struct SwissReaderX86
{
public static bool TryGet(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
if (Sse2.MoveMask(cmp_empty) != 0)
break;
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs
+9 -6
View File
@@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory
public partial class L4 public partial class L4
{ {
private unsafe class TLSF private unsafe struct TLSF
{ {
public const int MaxCacheLineCount = 8192; public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf; public static readonly nuint RequestedAllocation =
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
private readonly short[] _bucket;
private readonly CacheLineMetadata* _spareMemory; private readonly CacheLineMetadata* _spareMemory;
private readonly int _align; private readonly int _align;
private readonly int _maxLevel; private readonly int _maxLevel;
private int _metadataEnd; private int _metadataEnd;
private readonly Ptr64 _basePtr; private readonly Ptr64 _basePtr;
private fixed short _bucket[64];
private readonly uint _maxAllocSize; private readonly uint _maxAllocSize;
private readonly uint _totalSize; private readonly uint _totalSize;
@@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U)); _maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue; _maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize); _maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2];
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize); _spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0; _metadataEnd = 0;
@@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory
idx = candidate->NextLevel; idx = candidate->NextLevel;
} }
} }
for (int i = level + 1; i < _bucket.Length; i++) for (int i = level + 1; i < _bucket[63]; i++)
{ {
int idx = _bucket[i]; int idx = _bucket[i];
if (idx != 0) if (idx != 0)
src/Nitload.Common/HighPerformance/Memory/L4.cs
+25 -132
View File
@@ -5,49 +5,20 @@
// SPDX-License-Identifier: MIT // SPDX-License-Identifier: MIT
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using Nitload.Common.HighPerformance.ProcessorAbstraction; using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory namespace Nitload.Common.HighPerformance.Memory
{ {
public sealed unsafe partial class L4 : IDisposable public sealed unsafe partial class L4 : IDisposable
{ {
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096; private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
/// <summary>
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
/// </summary>
private readonly CacheLineMetadata* _head;
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
private readonly CacheLineDescription* _prediction;
private bool _disposed; private bool _disposed;
private bool _exhausted;
private int _aheadCount;
private readonly int _bindedNuma; private readonly int _bindedNuma;
private volatile int _prefetching;
private readonly nuint _baseAddress; private readonly nuint _baseAddress;
private readonly SwissTable _map; private SwissTable _addressMap;
private TLSF _bufferAllocator;
private readonly TLSF _bufferAllocator;
public L4( public L4(
int bindedNuma, int bindedNuma,
@@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize); ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize); ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma; _bindedNuma = bindedNuma;
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
// init order is fallow by align requirement of each component.
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize; ByteCapacity = normalizedBufferSize;
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize; AllocSize = swissBufferOffset + (nuint)BufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize); _baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress); _bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
_buffer += normalizedBufferSize; _addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
_head = (CacheLineMetadata*)ptr;
ptr += headSize;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
for (uint i = 1; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(0);
}
} }
public nuint ByteCapacity { get; init; } public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; } public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design",
"CA1063:Implement IDisposable correctly",
Justification = """
Only owns native memory via Numa.Free();
no managed IDisposable fields.
Dispose(bool) pattern is unnecessary.
""")]
public void Dispose() public void Dispose()
{ {
if (_disposed) { if (_disposed) {
return; return;
} }
_disposed = true; _disposed = true;
_map.Dispose(); _addressMap.Dispose();
#pragma warning disable CA1816
GC.SuppressFinalize(_bufferAllocator);
#pragma warning restore CA1816
// init order is fallow by align requirement of each component.
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize); Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
GC.SuppressFinalize(this); GC.SuppressFinalize(this);
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] internal void Collect()
public Handle Map( {
ref CacheLineDescription prediction ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Collect();
}
internal void Free(
int handle
) )
{ {
if ((_aheadCount < PredictionLeastAhead) && !_exhausted) ObjectDisposedException.ThrowIf(_disposed, this);
{ _bufferAllocator.Free(handle);
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{
try
{
if (_aheadCount < PredictionLeastAhead)
{
// do prefetch in work load now, sync
Prefetch();
}
}
finally
{
_prefetching = 0;
}
}
}
while (true)
{
if (SwissTable.Reader.TryGet(_map, prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
}
} }
private partial void Prefetch(); internal bool TryAllocate(
uint size,
out int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _bufferAllocator.TryAlloc(size, out handle);
}
} }
} }
test/STgSharpTest/UnitTest1.cs
-17
View File
@@ -32,7 +32,6 @@ using StgSharp;
using StgSharp.Collections; using StgSharp.Collections;
using StgSharp.Script; using StgSharp.Script;
using StgSharp.Script.Express; using StgSharp.Script.Express;
using Nitload.Common.Collections;
//using StgSharp.Model; //using StgSharp.Model;
@@ -47,22 +46,6 @@ namespace STgSharpTest
{ {
string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp"; string testExpPath = @"D:\Programmer\stepcode\data\ap203\ap203.exp";
[TestMethod]
public void SwissTableOperationsAreNotImplemented()
{
using SwissTable table = SwissTable.Create();
nuint key = 17;
Assert.ThrowsException<NotImplementedException>(
() => SwissTable.Reader.TryGet(table, key, out _));
Assert.ThrowsException<NotImplementedException>(
() => SwissTable.Writer.TryAdd(table, key, 100));
Assert.ThrowsException<NotImplementedException>(
() => SwissTable.Writer.TrySet(table, key, 100));
Assert.ThrowsException<NotImplementedException>(
() => SwissTable.Writer.TryRemove(table, key, out _));
}
[TestMethod] [TestMethod]
public void RegexTest() public void RegexTest()
{ {
test/StgSharpDebug/TestSwiss.cs
-46
View File
@@ -1,46 +0,0 @@
// -----------------------------------------------------------------------------
// file="TestSwiss"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
namespace Nitload
{
internal static partial class Program
{
public static void TestSwiss()
{
SwissTable swiss = SwissTable.Create();
for (nuint i = 0; i < 5000; i++)
{
Console.Write($"Set {i}: ");
_ = SwissTable.Writer.TrySet(swiss, i, i * 2);
Console.WriteLine($"Set a kvp of ({i},{i * 2})");
}
Random rand = new Random();
for (int i = 0; i < 100; i++)
{
Console.Write($"Get {i}: ");
nuint key = (nuint)rand.Next(4000, 5000);
if (SwissTable.Reader.TryGet(swiss, key, out nuint value))
{
Console.WriteLine($"Success in get a kvp of ({key},{value})");
} else
{
Console.WriteLine($"Failed to get a kvp of ({key},?)");
}
}
}
}
}