youmuliu/stgsharp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

before merge

Browse Source
This commit is contained in:
youmuliu
2026-10-12 04:13:46 +08:00
parent e24efd9f9c 3a2b123478
commit 2bef14493f
27 changed files with 797 additions and 1607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Nitload.Benchmark/SwissBenchmark.cs
-485
View File
@@ -1,485 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissBenchmark"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using BenchmarkDotNet.Attributes;
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace Nitload.Benchmark
{
/// <summary>
/// Benchmark comparing SwissTable against <see cref="Dictionary{TKey, TValue}" /> under
/// realistic L4 software cache address-mapping workloads. <para>/// L4 scenario
/// characteristics: /// - Keys are HLSF/malloc pointer addresses (large, 1024-byte aligned)
/// /// - Access pattern is Zipfian-skewed (20% hot cache lines receive 80% of lookups) ///
/// - Steady state: high hit-rate lookups dominate, with rare evict+refill on miss /// -
/// Working set: 512–4096 active cache line mappings /// </para>
/// </summary>
[MemoryDiagnoser]
public class SwissTableBench
{
/// <summary>
/// L4 cache line size in bytes (per L4_design.md).
/// </summary>
private const int CacheLineSize = 1024;
/// <summary>
/// Simulated heap base address for pointer-like keys.
/// </summary>
private const long HeapBase = 0x7FFF_0000_0000L;
/// <summary>
/// Number of lookup operations per Lookup benchmark call.
/// </summary>
private const int LookupIterations = 8192;
/// <summary>
/// Fraction of steady-state ops that are cache misses.
/// </summary>
private const double MissRate = 0.05;
/// <summary>
/// Number of steady-state operations per call. Each op is either a lookup (hit) or an
/// evict+insert+lookup (miss).
/// </summary>
private const int SteadyStateOps = 4096;
/// <summary>
/// Indices into _residentKeys for eviction targets (cold entries, tail region).
/// </summary>
private int[] _evictOrder = null!;
/// <summary>
/// Additional keys not in the table, for triggering misses.
/// </summary>
private nuint[] _externalKeys = null!;
// ── Pre-generated key arrays (pointer-like, 1024-byte aligned) ──
/// <summary>
/// Keys for initial table fill (simulates cached addresses).
/// </summary>
private nuint[] _residentKeys = null!;
/// <summary>
/// Pre-built steady-state operation script. Each entry: positive index → lookup
/// _residentKeys[idx], negative value → cache miss trigger (evict + insert from
/// _externalKeys).
/// </summary>
private int[] _steadyStateScript = null!;
/// <summary>
/// Uniform random lookup sequence (baseline comparison).
/// </summary>
private nuint[] _uniformLookups = null!;
/// <summary>
/// Zipfian-skewed lookup sequence (hot 20% accessed 80% of time).
/// </summary>
private nuint[] _zipfianLookups = null!;
private Dictionary<nuint, nuint> _dictLookup = null!;
private Dictionary<nuint, nuint> _freshDict = null!;
// ── Fresh instances for insert/mixed (rebuilt each iteration) ──
private SwissTable _freshSwiss = null!;
// ── Pre-filled lookup targets ──
private SwissTable _swissLookup = null!;
/// <summary>
/// Active cache line count. Matches L4 capacity range: 5 MB / 1024B ≈ 5120, 10 MB /
/// 1024B ≈ 10240. 512 and 1024 test low-occupancy; 4096 tests near-capacity.
/// </summary>
[Params(512, 1024, 4096)]
public int KeyCount;
[GlobalCleanup]
public void Cleanup()
{
_dictLookup.Clear();
_dictLookup.TrimExcess();
}
// ════════════════════════════════════════════════════════════
// Insert — bulk fill with scattered pointer addresses
// Simulates L4 initial cache warm-up phase.
// ════════════════════════════════════════════════════════════
[Benchmark]
public void Dictionary_Insert()
{
Dictionary<nuint, nuint> dict = _freshDict;
nuint[] keys = _residentKeys;
for (int i = 0; i < keys.Length; i++) {
dict[keys[i]] = keys[i] ^ unchecked((nuint)0xDEAD_BEEF);
}
}
// ════════════════════════════════════════════════════════════
// Lookup — Uniform (baseline comparison)
// ════════════════════════════════════════════════════════════
[Benchmark]
public nuint Dictionary_Lookup_Uniform()
{
nuint sum = 0;
nuint[] keys = _uniformLookups;
Dictionary<nuint, nuint> dict = _dictLookup;
for (int i = 0; i < keys.Length; i++)
{
if (dict.TryGetValue(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// Lookup — Zipfian (L4 hot path, skewed access)
// 20% of cache lines receive 80% of accesses.
// This is the most representative L4 benchmark.
// ════════════════════════════════════════════════════════════
[Benchmark(Baseline = true)]
public nuint Dictionary_Lookup_Zipfian()
{
nuint sum = 0;
nuint[] keys = _zipfianLookups;
Dictionary<nuint, nuint> dict = _dictLookup;
for (int i = 0; i < keys.Length; i++)
{
if (dict.TryGetValue(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// SteadyState — the KEY L4 benchmark
// Pre-filled table at capacity, then 4096 operations:
// ~95% are lookups (cache hits)
// ~5% are miss → evict cold entry → insert new entry → lookup new
// This directly simulates L4's main runtime loop.
// ════════════════════════════════════════════════════════════
[Benchmark]
public nuint Dictionary_SteadyState()
{
nuint sum = 0;
Dictionary<nuint, nuint> dict = _freshDict;
nuint[] resident = _residentKeys;
nuint[] external = _externalKeys;
int[] script = _steadyStateScript;
int[] evict = _evictOrder;
int evictCursor = 0;
int externalCursor = 0;
for (int i = 0; i < script.Length; i++)
{
int op = script[i];
if (op >= 0)
{
// Cache hit: lookup existing entry
if (dict.TryGetValue(resident[op], out nuint value))
{
sum += value;
}
} else
{
// Cache miss: evict a cold entry, insert a new one, then lookup
int evictIdx = evict[evictCursor % evict.Length];
evictCursor++;
dict.Remove(resident[evictIdx]);
int extIdx = externalCursor % external.Length;
externalCursor++;
nuint newKey = external[extIdx];
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
dict[newKey] = newVal;
if (dict.TryGetValue(newKey, out nuint v)) {
sum += v;
}
}
}
return sum;
}
[IterationSetup(Targets = [
nameof(SwissTable_Insert),
nameof(SwissTable_SteadyState),
nameof(Dictionary_Insert),
nameof(Dictionary_SteadyState)
])]
public void IterationSetup()
{
_freshSwiss = SwissTable.Create();
_freshDict = new Dictionary<nuint, nuint>(KeyCount);
// Pre-fill for steady-state benchmarks
for (int i = 0; i < KeyCount; i++)
{
nuint key = _residentKeys[i];
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF);
_freshSwiss.TryAddOrSet(key, val, out _);
_freshDict[key] = val;
}
}
// ════════════════════════════════════════════════════════════
// Setup / Teardown
// ════════════════════════════════════════════════════════════
[GlobalSetup]
public void Setup()
{
Random rng = new Random(42);
// ── Generate pointer-like keys (HeapBase + random_offset * 1024) ──
_residentKeys = GeneratePointerKeys(KeyCount, rng);
_externalKeys = GeneratePointerKeys(KeyCount, rng); // disjoint set for miss simulation
// ── Build lookup sequences ──
_zipfianLookups = GenerateZipfianLookups(_residentKeys, LookupIterations, rng);
_uniformLookups = GenerateUniformLookups(_residentKeys, LookupIterations, rng);
// ── Build steady-state operation script ──
_steadyStateScript = GenerateSteadyStateScript(KeyCount, SteadyStateOps, MissRate, rng);
// ── Build eviction order (cold tail entries first) ──
_evictOrder = new int[KeyCount];
for (int i = 0; i < KeyCount; i++) {
_evictOrder[i] = i;
}
// Shuffle so eviction isn't just sequential
Shuffle(_evictOrder, rng);
// ── Pre-fill lookup tables ──
_swissLookup = SwissTable.Create();
_dictLookup = new Dictionary<nuint, nuint>(KeyCount);
for (int i = 0; i < KeyCount; i++)
{
nuint key = _residentKeys[i];
nuint val = key ^ unchecked((nuint)0xDEAD_BEEF); // simulated cache pointer
_swissLookup.TryAddOrSet(key, val, out _);
_dictLookup[key] = val;
}
}
[Benchmark]
public void SwissTable_Insert()
{
SwissTable swiss = _freshSwiss;
nuint[] keys = _residentKeys;
for (int i = 0; i < keys.Length; i++) {
swiss.TryAddOrSet(keys[i], keys[i] ^ unchecked((nuint)0xDEAD_BEEF), out _);
}
}
[Benchmark]
public nuint SwissTable_Lookup_Uniform()
{
nuint sum = 0;
nuint[] keys = _uniformLookups;
SwissTable swiss = _swissLookup;
for (int i = 0; i < keys.Length; i++)
{
if (swiss.TryGet(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
[Benchmark]
public nuint SwissTable_Lookup_Zipfian()
{
nuint sum = 0;
nuint[] keys = _zipfianLookups;
SwissTable swiss = _swissLookup;
for (int i = 0; i < keys.Length; i++)
{
if (swiss.TryGet(keys[i], out nuint value)) {
sum += value;
}
}
return sum;
}
[Benchmark]
public nuint SwissTable_SteadyState()
{
nuint sum = 0;
SwissTable swiss = _freshSwiss;
nuint[] resident = _residentKeys;
nuint[] external = _externalKeys;
int[] script = _steadyStateScript;
int[] evict = _evictOrder;
int evictCursor = 0;
int externalCursor = 0;
for (int i = 0; i < script.Length; i++)
{
int op = script[i];
if (op >= 0)
{
// Cache hit: lookup existing entry
if (swiss.TryGet(resident[op], out nuint value))
{
sum += value;
}
} else
{
// Cache miss: evict a cold entry, insert a new one, then lookup
int evictIdx = evict[evictCursor % evict.Length];
evictCursor++;
swiss.TryRemove(resident[evictIdx], out _);
int extIdx = externalCursor % external.Length;
externalCursor++;
nuint newKey = external[extIdx];
nuint newVal = newKey ^ unchecked((nuint)0xDEAD_BEEF);
swiss.TryAddOrSet(newKey, newVal, out _);
if (swiss.TryGet(newKey, out nuint v)) {
sum += v;
}
}
}
return sum;
}
// ════════════════════════════════════════════════════════════
// Data generation helpers
// ════════════════════════════════════════════════════════════
/// <summary>
/// Generate <paramref name="count" /> unique pointer-like keys. Each key = HeapBase +
/// randomOffset * CacheLineSize, simulating HLSF-allocated 1024-byte aligned addresses
/// scattered across a ~1 GB virtual address range.
/// </summary>
private static nuint[] GeneratePointerKeys(
int count,
Random rng
)
{
nuint[] keys = new nuint[count];
HashSet<long> used = new HashSet<long>(count);
for (int i = 0; i < count; i++)
{
long offset;
do
{
// Random offset within ~1M cache lines (1 GB range)
offset = (long)rng.Next(0, 1 << 20) * CacheLineSize;
} while (!used.Add(offset));
keys[i] = (nuint)(HeapBase + offset);
}
return keys;
}
/// <summary>
/// Generate a steady-state operation script. Each entry: non-negative value = index into
/// _residentKeys (lookup/hit), negative value (-1) = cache miss trigger (evict → insert →
/// lookup). Lookup indices are Zipfian-skewed.
/// </summary>
private static int[] GenerateSteadyStateScript(
int keyCount,
int opCount,
double missRate,
Random rng
)
{
int hotSetSize = Math.Max(1, keyCount / 5);
int[] script = new int[opCount];
for (int i = 0; i < opCount; i++)
{
if (rng.NextDouble() < missRate)
{
script[i] = -1; // miss
} else
{
// Zipfian hit
if (rng.NextDouble() < 0.8)
{
script[i] = rng.Next(0, hotSetSize);
} else
{
script[i] = rng.Next(hotSetSize, keyCount);
}
}
}
return script;
}
/// <summary>
/// Generate uniform random lookup sequence (baseline for comparison with Zipfian).
/// </summary>
private static nuint[] GenerateUniformLookups(
nuint[] sourceKeys,
int count,
Random rng
)
{
nuint[] lookups = new nuint[count];
for (int i = 0; i < count; i++) {
lookups[i] = sourceKeys[rng.Next(0, sourceKeys.Length)];
}
return lookups;
}
/// <summary>
/// Generate Zipfian-skewed lookup sequence: 80% of accesses hit the hottest 20% of keys
/// (sorted by index as "hotness proxy"), 20% of accesses hit the remaining 80%. This
/// models matrix computation where a small panel working set is reused heavily.
/// </summary>
private static nuint[] GenerateZipfianLookups(
nuint[] sourceKeys,
int count,
Random rng
)
{
int hotSetSize = Math.Max(1, sourceKeys.Length / 5);
nuint[] lookups = new nuint[count];
for (int i = 0; i < count; i++)
{
if (rng.NextDouble() < 0.8)
{
lookups[i] = sourceKeys[rng.Next(0, hotSetSize)];
} else
{
lookups[i] = sourceKeys[rng.Next(hotSetSize, sourceKeys.Length)];
}
}
return lookups;
}
/// <summary>
/// Fisher-Yates shuffle.
/// </summary>
private static void Shuffle(
int[] array,
Random rng
)
{
for (int i = array.Length - 1; i > 0; i--)
{
int j = rng.Next(0, i + 1);
(array[i], array[j]) = (array[j], array[i]);
}
}
}
}
src/Nitload.Common/Collections/SwissTable.X86.cs
-130
View File
@@ -1,130 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissTable.X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.Collections
{
public partial class SwissTable
{
private unsafe class BucketScannerX86 : IBucketScanner
{
public bool TryFind(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector);
int mask = Sse2.MoveMask(cmpResult);
while (mask != 0)
{
int bitPos = BitOperations.TrailingZeroCount(mask);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= mask - 1;
}
value = default;
return false;
}
public bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
bucket->Control[index] = 0xFE; // mark as DELETED
// no need to remove Key/Value
return true;
}
matches &= matches - 1;
}
value = default;
return false;
}
public bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0)
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128(
(byte*)&bucket->Control);
// === Phase 1: Look up an existing key ===
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
// Key already exists -> update value
bucket->Values[index] = value;
isNewKey = false;
return true;
}
matches &= matches - 1;
}
// === Phase 2: Track an available insertion slot ===
int emptyOrDeleted = Sse2.MoveMask(bucketCtrl);
if (emptyOrDeleted == 0)
{
isNewKey = false;
return false; // bucket full
}
int slot = BitOperations.TrailingZeroCount(emptyOrDeleted);
bucket->Control[slot] = ctrl;
bucket->Keys[slot] = key;
bucket->Values[slot] = value;
isNewKey = true;
return true;
}
}
}
}
src/Nitload.Common/Collections/SwissTable.cs
-224
View File
@@ -1,224 +0,0 @@
// -----------------------------------------------------------------------------
// file="SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.Collections
{
/// <summary>
///
/// </summary>
public sealed unsafe partial class SwissTable : IDisposable
{
private const int BucketCount = 1 << EntryMaskWidth;
private const int BucketSize = 16;
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<Entry>();
private readonly Entry* _entries;
private readonly Action<nuint> _freeHandle;
private bool _disposed;
private SwissTable(
Entry* buffer,
Action<nuint> freeHandle
)
{
_entries = buffer;
_freeHandle = freeHandle;
}
public static SwissTable Create()
{
Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf<Entry>(), 16);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle));
}
public static SwissTable Create(
Func<nuint, nuint> allocator,
Action<nuint> freeHandle
)
{
Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, freeHandle);
}
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_freeHandle((nuint)_entries);
GC.SuppressFinalize(this);
}
public bool TryAddOrSet(
nuint key,
nuint value,
out bool isNewKey
)
{
uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey);
}
public bool TryGet(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryFind(key, hash, bucket, out value);
/*
* hash = CRC32 of key
* entry = low 8 of hash
* ctrl = 8-15 of hash
* index = rest of hash
* find value of hash
*/
}
public bool TryRemove(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryRemove(key, hash, bucket, out value);
}
internal static SwissTable Create(
nuint bufferPtr,
Action<nuint> FreeHandle
)
{
Entry* buffer = (Entry*)bufferPtr;
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, FreeHandle);
}
~SwissTable()
{
Dispose();
}
[InlineArray(BucketCount)]
private struct Entry
{
public Bucket Bucket;
}
private unsafe struct Bucket
{
public BucketControl Control;
public BucketKeyStorage Keys;
public BucketValueStorage Values;
}
private interface IBucketScanner
{
static IBucketScanner Current { get; } = Create();
bool TryFind(
nuint key,
uint Hash,
Bucket* bucket,
out nuint value
);
unsafe bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
);
bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
);
private static IBucketScanner Create()
{
if (Sse42.IsSupported)
{
return new BucketScannerX86();
} else if (ArmBase.IsSupported)
{
throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform.");
} else
{
throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform.");
}
}
}
[InlineArray(BucketSize)]
private struct BucketControl
{
public byte Control;
}
[InlineArray(BucketSize)]
private struct BucketKeyStorage
{
public nuint value;
}
[InlineArray(BucketSize)]
private struct BucketValueStorage
{
public nuint value;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.CacheLine.cs
+5 -25
View File
@@ -5,7 +5,6 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory
@@ -23,15 +22,11 @@ namespace Nitload.Common.HighPerformance.Memory
}
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata
private struct CacheLineMetadata
{
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(24)] internal nuint Origin;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
@@ -50,32 +45,17 @@ namespace Nitload.Common.HighPerformance.Memory
#endregion
#region mask
[FieldOffset(16)] internal AllocationState State;
[FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion
}
#pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit)]
public unsafe struct Handle : IDisposable
[StructLayout(LayoutKind.Explicit, Size = 16)]
public struct Handle : IDisposable
{
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
[FieldOffset(0)] private nuint _handle;
private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{
_handle = handle;
_head = cacheRefCount;
}
[FieldOffset(8)] private nuint _reserved;
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
public Handle() { }
@@ -88,7 +68,7 @@ namespace Nitload.Common.HighPerformance.Memory
return;
}
_handle = 0;
_ = Interlocked.Decrement(ref _head->RefCount);
_reserved = 0;
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Debug.cs
-92
View File
@@ -1,92 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Debug"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
/**/
public partial class L4
{
public void DebugAgeAll()
{
// AgeAll();
}
public bool DebugContains(
nuint origin
)
{
return _map.TryGet(origin, out _);
}
public int DebugEvict(
int count = 1
)
{
return Evict(count);
}
public unsafe DebugSnapshot GetDebugSnapshot()
{
throw new NotImplementedException();
/*
int maxId = _maxId;
byte[] rrpv = new byte[maxId];
byte* p = (byte*)_eviction;
for (int i = 0; i < maxId; i++) {
rrpv[i] = p[i];
}
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
/**/
}
public readonly struct DebugSnapshot
{
/// <summary>
/// RRPV value for each line（0~3）, length == MaxId
/// </summary>
public readonly byte[] RrpvSnapshot;
/// <summary>
/// Count of activated cache line handle
/// </summary>
public readonly int ActiveCount;
/// <summary>
/// Count of predictor ahead of current map position（aheadCount）
/// </summary>
public readonly int AheadCount;
/// <summary>
/// Max ID of the line being using or once used（maxId）
/// </summary>
public readonly int MaxId;
internal DebugSnapshot(
int activeCount,
int aheadCount,
int maxId,
byte[] rrpv
)
{
ActiveCount = activeCount;
AheadCount = aheadCount;
MaxId = maxId;
RrpvSnapshot = rrpv;
}
}
}
/**/
}
src/Nitload.Common/HighPerformance/Memory/L4.Evict.cs
-64
View File
@@ -1,64 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
/// <summary>
///
/// </summary>
/// <param name="count">
///
/// </param>
/// <returns>
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(
int count = 1
)
{
count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count;
for (int i = 0; i < CacheLineCount; i++)
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i) ? 1 : 0;
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index
)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
return true;
}
return false;
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Exceptions.cs
-34
View File
@@ -1,34 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Exceptions"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace Nitload.Common.HighPerformance.Memory
{
#pragma warning disable CA1032 // Implement standard exception constructors
public sealed class L4PredictUnregisteredException : Exception
{
public L4PredictUnregisteredException(
int id
)
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
}
public sealed class L4PredictConflictException : Exception
{
public L4PredictConflictException(
nuint basePtr,
ushort policy,
int current
)
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
}
#pragma warning restore CA1032 // Implement standard exception constructors
}
src/Nitload.Common/HighPerformance/Memory/L4.Predict.cs
-38
View File
@@ -1,38 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private IL4Predict CurrentPredict { get; set; }
public void ResetPredict(
IL4Predict predict
)
{
if (CurrentPredict is null)
{
CurrentPredict = predict;
return;
}
// TODO: write back all and register new predictor
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Predictor.cs
-146
View File
@@ -1,146 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Predictor"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace Nitload.Common.HighPerformance.Memory
{
[Flags]
public enum PredictResult
{
Success = 1,
Exhausted = 2,
}
public interface IL4Predict
{
PredictResult Predict(
Span<CacheLineDescription> node,
out int count
);
/// <summary>
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
/// name="cache" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Destination cache line buffer.
/// </param>
void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
);
/// <summary>
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Source cache line buffer.
/// </param>
void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
);
}
[StructLayout(LayoutKind.Explicit)]
public struct CacheLineDescription : IEquatable<CacheLineDescription>
{
public override readonly bool Equals(
object? obj
)
{
return (obj is CacheLineDescription prediction) &&
Address.Equals(prediction.Address) &&
(MapPolicy == prediction.MapPolicy);
}
public readonly bool Equals(
CacheLineDescription other
)
{
return
(Address == other.Address) && (MapPolicy == other.MapPolicy);
}
public override readonly int GetHashCode()
{
return (int)ComposePredictionKey64(Address, MapPolicy);
}
public readonly nuint GetHashCodeLong()
{
return ComposePredictionKey64(Address, MapPolicy);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint ComposePredictionKey64(
nuint address,
ushort mapPolicy
)
{
nuint x = address;
nuint y = mapPolicy;
unchecked
{
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
mixed ^= mixed >> 30;
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
mixed ^= mixed >> 27;
mixed *= (nuint)0x94d049bb133111ebUL;
mixed ^= mixed >> 31;
return mixed;
}
}
public static bool operator !=(
CacheLineDescription left,
CacheLineDescription right
)
{
return !(left == right);
}
public static bool operator ==(
CacheLineDescription left,
CacheLineDescription right
)
{
return left.Equals(right);
}
#pragma warning disable CA1051
[FieldOffset(0)]public nuint Address;
[FieldOffset(8)]public uint Size;
[FieldOffset(12)]public ushort MapPolicy;
[FieldOffset(14)]internal short Predictor;
#pragma warning restore CA1051
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Prefetch.cs
-102
View File
@@ -1,102 +0,0 @@
// -----------------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.ComponentModel;
using System.Numerics;
namespace Nitload.Common.HighPerformance.Memory
{
public unsafe partial class L4
{
private partial void Prefetch()
{
if (CurrentPredict is null) {
return;
}
IL4Predict predict = CurrentPredict;
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
_exhausted = (result & PredictResult.Exhausted) != 0;
if (actualCount <= 0)
{
_exhausted = true;
return;
}
int missIndex = 0;
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!_map.TryGet(address, out _))
{
// miss and record
span[missIndex] = handle;
missIndex++;
}
}
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
{
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
_predictCount[index] = 0; // reset eviction counter
#region update predictor belonging
if (_predictCount[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref _predictCount[index]);
int mapCount = Volatile.Read(ref _mapCount[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref _mapCount[index], -decrement);
}
_predictCount[index]++;
#endregion
continue;
}
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.SwissTable.cs
+217
View File
@@ -0,0 +1,217 @@
// -----------------------------------------------------------------------------
// file="L4.SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4
{
private const int BucketCapacity = 16;
private const uint CrcMagic = 0x7f1c9961;
private const int GroupCount = 1 << GroupCountLog;
private const int GroupCountLog = 10;
private const int IdealCount = GroupCount * BucketCapacity;//16384
private const int OffsetOfBucket = IdealCount;
private static readonly int TableBufferSize =
IdealCount + (GroupCount * Unsafe.SizeOf<SwissBucket>());
internal static readonly int BufferSize = TableBufferSize * 2;
private enum SwissControl : byte
{
Empty = 0b10000000,
Deleted = 0b10000001
}
[StructLayout(LayoutKind.Sequential)]
internal struct AddressSource
{
public ulong Address;
public ulong Description;
public override readonly bool Equals(
object obj
)
{
return obj is AddressSource source && this == source;
}
public override readonly int GetHashCode()
{
return HashCode.Combine(Address, Description);
}
public static bool operator !=(
AddressSource left,
AddressSource right
)
{
return !(left == right);
}
public static bool operator ==(
AddressSource left,
AddressSource right
)
{
return left.Address == right.Address && left.Description == right.Description;
}
}
[InlineArray(BucketCapacity)]
internal struct KeyArray
{
private AddressSource _element0;
}
internal struct SwissBucket
{
public KeyArray Keys;
public fixed ulong Values[BucketCapacity];
}
internal sealed unsafe class SwissTable : IDisposable
{
private byte* _buffer_read;
private byte* _buffer_write;
private readonly ReaderWriterLockSlim _synchronization = new(LockRecursionPolicy.NoRecursion);
private SwissTable(byte* buffers)
{
_buffer_read = buffers;
_buffer_write = buffers + TableBufferSize;
InitializeBuffer(_buffer_read);
InitializeBuffer(_buffer_write);
}
internal static SwissTable Create(byte* buffers)
{
ArgumentNullException.ThrowIfNull(buffers);
return new SwissTable(buffers);
}
/// <summary>
/// Reads the published buffer. Callers must prevent a swap while the read is in progress.
/// </summary>
public bool TryGet(AddressSource key, out ulong value)
{
_synchronization.EnterReadLock();
try
{
return SwissReaderX86.TryGet(_buffer_read, key, out value);
}
finally
{
_synchronization.ExitReadLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryAddOrSet(AddressSource key, ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryAddOrSet(_buffer_write, key, value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Writes to the inactive buffer. Calls must be serialized by the caller.
/// </summary>
public bool TryRemove(AddressSource key, out ulong value)
{
_synchronization.EnterWriteLock();
try
{
return SwissWriterX86.TryRemove(_buffer_write, key, out value);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Clears the inactive buffer. Call only after all readers of its previous generation have quiesced.
/// </summary>
public void PrepareWriteBuffer()
{
_synchronization.EnterWriteLock();
try
{
InitializeBuffer(_buffer_write);
}
finally
{
_synchronization.ExitWriteLock();
}
}
/// <summary>
/// Publishes the write buffer. The caller must provide the phase barrier around this operation.
/// </summary>
public void SwapBuffers()
{
_synchronization.EnterWriteLock();
try
{
byte* temporary = _buffer_read;
_buffer_read = _buffer_write;
_buffer_write = temporary;
}
finally
{
_synchronization.ExitWriteLock();
}
}
public void Dispose()
{
_synchronization.EnterWriteLock();
try
{
_buffer_read = null;
_buffer_write = null;
}
finally
{
_synchronization.ExitWriteLock();
}
_synchronization.Dispose();
}
private static void InitializeBuffer(byte* buffer)
{
Unsafe.InitBlock(buffer, (byte)SwissControl.Empty, (uint)IdealCount);
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Swiss_X86.cs
+200
View File
@@ -0,0 +1,200 @@
// -----------------------------------------------------------------------------
// file="L4.Swiss_X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace Nitload.Common.HighPerformance.Memory
{
public partial class L4
{
private readonly unsafe struct SwissWriterX86
{
public static bool TryAddOrSet(
byte* buffer,
AddressSource key,
ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
uint first_tomb_group = 0;
int first_tomb_idx = -1;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte>* control_bytes_ptr = control_bytes + begin;
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)control_bytes_ptr);
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> target_tomb_v128 = Vector128.Create((byte)SwissControl.Deleted);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
Vector128<byte> cmp_tomb = Sse2.CompareEqual(control_bytes_v128, target_tomb_v128);
int hash_mask = Sse2.MoveMask(cmp_hash);
int empty_mask = Sse2.MoveMask(cmp_empty);
int tomb_mask = Sse2.MoveMask(cmp_tomb);
if (tomb_mask != 0 && first_tomb_idx < 0)
{
int index = BitOperations.TrailingZeroCount(tomb_mask);
first_tomb_group = begin;
first_tomb_idx = index;
}
while (hash_mask != 0)
{
int index = BitOperations.TrailingZeroCount(hash_mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
bucket->Values[index] = value;
return true;
}
hash_mask &= ~(1 << index);
}
while (empty_mask != 0)
{
int index = BitOperations.TrailingZeroCount(empty_mask);
if (first_tomb_idx < 0)
{
SwissBucket* bucket = &buckets[begin];
bucket->Keys[index] = key;
bucket->Values[index] = value;
((byte*)control_bytes_ptr)[index] = target_hash;
} else
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
}
return true;
}
begin = (begin + 1) & (GroupCount - 1);
}
if (first_tomb_idx >= 0)
{
SwissBucket* bucket = &buckets[first_tomb_group];
bucket->Keys[first_tomb_idx] = key;
bucket->Values[first_tomb_idx] = value;
((byte*)(control_bytes + first_tomb_group))[first_tomb_idx] = target_hash;
return true;
}
value = 0;
return false;
}
public static bool TryRemove(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
((byte*)(control_bytes + begin))[index] = (byte)SwissControl.Deleted;
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
int empty_mask = Sse2.MoveMask(cmp_empty);
if (empty_mask != 0)
{
break;
}
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
private readonly unsafe struct SwissReaderX86
{
public static bool TryGet(
byte* buffer,
AddressSource key,
out ulong value
)
{
Vector128<byte>* control_bytes = (Vector128<byte>*)buffer;
SwissBucket* buckets = (SwissBucket*)(buffer + IdealCount);
uint hash = Sse42.Crc32(CrcMagic, (uint)key.GetHashCode());
uint begin = hash % GroupCount;
byte target_hash = (byte)((hash >> GroupCountLog) & ~(1 << 7));
for (int i = 0; i < GroupCount; i++)
{
Vector128<byte> control_bytes_v128 = Sse2.LoadAlignedVector128((byte*)(control_bytes + begin));
Vector128<byte> target_hash_v128 = Vector128.Create(target_hash);
Vector128<byte> cmp_hash = Sse2.CompareEqual(control_bytes_v128, target_hash_v128);
int mask = Sse2.MoveMask(cmp_hash);
while (mask != 0)
{
int index = BitOperations.TrailingZeroCount(mask);
SwissBucket* bucket = &buckets[begin];
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= ~(1 << index);
}
Vector128<byte> target_empty_v128 = Vector128.Create((byte)SwissControl.Empty);
Vector128<byte> cmp_empty = Sse2.CompareEqual(control_bytes_v128, target_empty_v128);
if (Sse2.MoveMask(cmp_empty) != 0)
break;
begin = (begin + 1) & (GroupCount - 1);
}
value = 0;
return false;
}
}
}
}
src/Nitload.Common/HighPerformance/Memory/L4.Tlsf.cs
+9 -6
View File
@@ -19,20 +19,21 @@ namespace Nitload.Common.HighPerformance.Memory
public partial class L4
{
private unsafe class TLSF
private unsafe struct TLSF
{
public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
private readonly short[] _bucket;
public static readonly nuint RequestedAllocation =
((nuint)MaxCacheLineCount + 1u) * CacheLineMetadata.SizeOf;
private readonly CacheLineMetadata* _spareMemory;
private readonly int _align;
private readonly int _maxLevel;
private int _metadataEnd;
private readonly Ptr64 _basePtr;
private fixed short _bucket[64];
private readonly uint _maxAllocSize;
private readonly uint _totalSize;
@@ -48,7 +49,9 @@ namespace Nitload.Common.HighPerformance.Memory
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2];
// _bucket = new short[GetLevel(_maxAllocSize) + 2];
_bucket[63] = (short)(GetLevel(_maxAllocSize) + 2);
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0;
@@ -268,7 +271,7 @@ namespace Nitload.Common.HighPerformance.Memory
idx = candidate->NextLevel;
}
}
for (int i = level + 1; i < _bucket.Length; i++)
for (int i = level + 1; i < _bucket[63]; i++)
{
int idx = _bucket[i];
if (idx != 0)
src/Nitload.Common/HighPerformance/Memory/L4.cs
+26 -133
View File
@@ -5,49 +5,20 @@
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using Nitload.Common.HighPerformance.Memory;
using Nitload.Common.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
namespace Nitload.Common.HighPerformance.Memory
{
public sealed unsafe partial class L4 : IDisposable
{
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
/// <summary>
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
/// </summary>
private readonly CacheLineMetadata* _head;
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
private int _aheadCount;
private readonly int _bindedNuma;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SwissTable _map;
private readonly TLSF _bufferAllocator;
private SwissTable _addressMap;
private TLSF _bufferAllocator;
public L4(
int bindedNuma,
@@ -59,130 +30,52 @@ namespace Nitload.Common.HighPerformance.Memory
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1));
ByteCapacity = normalizedBufferSize;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
nuint swissBufferOffset = normalizedBufferSize + TLSF.RequestedAllocation;
AllocSize = swissBufferOffset + (nuint)BufferSize;
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
_buffer += normalizedBufferSize;
_head = (CacheLineMetadata*)ptr;
ptr += headSize;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
for (uint i = 1; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(0);
}
_addressMap = SwissTable.Create((byte*)(_baseAddress + swissBufferOffset));
}
public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design",
"CA1063:Implement IDisposable correctly",
Justification = """
Only owns native memory via Numa.Free();
no managed IDisposable fields.
Dispose(bool) pattern is unnecessary.
""")]
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_map.Dispose();
#pragma warning disable CA1816
GC.SuppressFinalize(_bufferAllocator);
#pragma warning restore CA1816
// init order is fallow by align requirement of each component.
_addressMap.Dispose();
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(
ref CacheLineDescription prediction
)
internal void Collect()
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{
try
{
if (_aheadCount < PredictionLeastAhead)
{
// do prefetch in work load now, sync
Prefetch();
}
}
finally
{
_prefetching = 0;
}
}
}
while (true)
{
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
}
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Collect();
}
private partial void Prefetch();
internal void Free(
int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
_bufferAllocator.Free(handle);
}
internal bool TryAllocate(
uint size,
out int handle
)
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _bufferAllocator.TryAlloc(size, out handle);
}
}
}
src/Nitload.Common/HighPerformance/ProcessorAbstraction/LogicalCore.cs
+9 -10
View File
@@ -16,22 +16,21 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
public readonly struct LogicalCore
{
public readonly uint GlobalCpuId;
public readonly uint NumaGroupId;
/// <summary>
/// Stays 0 on linux.
/// </summary>
public readonly uint ProcessorGroupId;
public readonly uint SocketId;
public readonly uint CpuIndex;
public readonly uint NumaNodeId;
internal LogicalCore(
uint globalCpuId,
uint numaGroupId,
uint processorGroupId,
uint socketId,
uint cpuIndex,
uint numaNodeId
)
{
GlobalCpuId = globalCpuId;
NumaGroupId = numaGroupId;
ProcessorGroupId = processorGroupId;
SocketId = socketId;
CpuIndex = cpuIndex;
NumaNodeId = numaNodeId;
}
src/Nitload.Common/HighPerformance/ProcessorAbstraction/Numa.cs
+49 -17
View File
@@ -33,7 +33,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
return NumaAllocNative(size, numaId);
@@ -69,7 +69,7 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
if ((numaId < 0) || !ContainsNode((uint)numaId)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
NumaFreeNative(address, size, numaId);
@@ -80,8 +80,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
public static partial int GetNumaEach(
uint numaId,
Span<ulong> CpuIndexInNuma,
Span<uint> globalCpuIdInNuma
Span<uint> processorGroupIds,
Span<uint> socketIds,
Span<uint> cpuIndices,
Span<uint> numaNodeIds
);
[LibraryImport(Native.LibName, EntryPoint = "nfn_get_numa_overall")]
@@ -97,10 +99,10 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
L4[] buffers = new L4[NumaNodeCount];
for (int i = 0; i < NumaNodeCount; i++)
L4[] buffers = new L4[_numaNodes.Length];
for (int i = 0; i < _numaNodes.Length; i++)
{
L4 buffer = new L4(i);
L4 buffer = new L4((int)_numaNodes[i].NumaNodeId);
buffers[i] = buffer;
}
return buffers;
@@ -108,14 +110,29 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
internal static void Initialize()
{
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
int numaNodeCount = 0;
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
if (0 == GetNumaOverall(ref numaNodeCount, default)) {
throw new InvalidOperationException("Failed to get NUMA overall information.");
}
_numaNodes = new NumaNode[numaNodeCount];
for (uint i = 0; i < numaNodeCount; i++) {
_numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]);
int[] coreCountInNuma = new int[numaNodeCount];
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
throw new InvalidOperationException("Failed to get NUMA node processor counts.");
}
int activeNodeCount = 0;
for (int i = 0; i < coreCountInNuma.Length; i++) {
if (coreCountInNuma[i] > 0) {
activeNodeCount++;
}
}
NumaNodeCount = activeNodeCount;
_numaNodes = new NumaNode[activeNodeCount];
int nodeIndex = 0;
for (uint numaId = 0; numaId < numaNodeCount; numaId++) {
int coreCount = coreCountInNuma[(int)numaId];
if (coreCount == 0) {
continue;
}
_numaNodes[nodeIndex++] = InitNumaNode(numaId, coreCount);
}
}
@@ -123,8 +140,8 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial void SetThreadAffinity(
int coreId,
int numaId
int cpuIndex,
int processorGroupId
);
[MethodImpl(MethodImplOptions.NoInlining)]
@@ -133,9 +150,24 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
int count
)
{
Span<uint> globalIndex = stackalloc uint[count];
_ = GetNumaEach(numaId, null, globalIndex);
return new(numaId, count, globalIndex);
Span<uint> processorGroupIds = stackalloc uint[count];
Span<uint> socketIds = stackalloc uint[count];
Span<uint> cpuIndices = stackalloc uint[count];
Span<uint> numaNodeIds = stackalloc uint[count];
if (0 == GetNumaEach(numaId, processorGroupIds, socketIds, cpuIndices, numaNodeIds)) {
throw new InvalidOperationException($"Failed to enumerate processors for NUMA node {numaId}.");
}
return new(numaId, count, processorGroupIds, socketIds, cpuIndices, numaNodeIds);
}
private static bool ContainsNode(uint numaId)
{
foreach (NumaNode node in _numaNodes) {
if (node.NumaNodeId == numaId) {
return true;
}
}
return false;
}
[LibraryImport(Native.LibName, EntryPoint = "nfn_numa_alloc")]
src/Nitload.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs
+12 -2
View File
@@ -22,16 +22,26 @@ namespace Nitload.Common.HighPerformance.ProcessorAbstraction
internal NumaNode(
uint numaId,
int count,
Span<uint> logicalCoreIds
Span<uint> processorGroupIds,
Span<uint> socketIds,
Span<uint> cpuIndices,
Span<uint> numaNodeIds
)
{
NumaNodeId = numaId;
_logicalCores = new LogicalCore[count];
for (int i = 0; i < count; i++) {
_logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]);
_logicalCores[i] = new LogicalCore(
processorGroupIds[i],
socketIds[i],
cpuIndices[i],
numaNodeIds[i]);
}
}
public uint NumaNodeId { get; }
public int LogicalCoreCount => _logicalCores.Length;
public ReadOnlySpan<LogicalCore> LogicalCores => _logicalCores;
src/Nitload.Common/Threading/ThreadHelper.cs
+3 -3
View File
@@ -37,7 +37,7 @@ namespace Nitload.Threading
LogicalCore core
)
{
SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId);
SetThreadAffinity(core.CpuIndex, core.ProcessorGroupId);
}
[Conditional("DEBUG")]
@@ -55,8 +55,8 @@ namespace Nitload.Threading
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static partial void SetThreadAffinity(
uint globalCoreId,
uint numaNodeId
uint cpuIndex,
uint processorGroupId
);
#pragma warning restore CA5393
src/Nitload.Common/Timing/StgSharpTime.cs
+1 -1
View File
@@ -97,7 +97,7 @@ namespace Nitload
}
Thread.Sleep(10);
Numa.SetThreadAffinity((int)threadConfig.BindedCoreIndex, (int)threadConfig.BindedNumaNode);
Numa.SetThreadAffinity((int)threadConfig.CpuIndex, (int)threadConfig.ProcessorGroupId);
while (mainProvider.IsRunning)
{
long elapsedTicks;
src/Nitload.Common/Timing/TimeSourceProvider.cs
+4 -4
View File
@@ -14,15 +14,15 @@ using System.Threading;
namespace Nitload.Common.Timing
{
public class TimeSourceStartupConfiguration(
nuint bindedNumaNode = 0,
nuint bindedCoreIndex = 0,
nuint processorGroupId = 0,
nuint cpuIndex = 0,
ThreadPriority threadPriority = ThreadPriority.Highest
)
{
public nuint BindedNumaNode { get; } = bindedNumaNode;
public nuint ProcessorGroupId { get; } = processorGroupId;
public nuint BindedCoreIndex { get; } = bindedCoreIndex;
public nuint CpuIndex { get; } = cpuIndex;
public ThreadPriority Priority { get; } = threadPriority;
src/Nitload.Native/.clangd
+1 -1
View File
@@ -1,2 +1,2 @@
CompileFlags:
CompilationDatabase: ../..
CompilationDatabase: .
src/Nitload.Native/README.md
+3 -3
View File
@@ -53,11 +53,11 @@ does not cross-compile the C library.
## clangd in VS Code
The `.clangd` file searches for `compile_commands.json` at the repository root.
The `.clangd` file searches for `compile_commands.json` in `src/Nitload.Native`.
When using CMake Tools, set `cmake.copyCompileCommands` to
`${workspaceFolder}/compile_commands.json` so the active Debug or Release preset
`${workspaceFolder}/src/Nitload.Native/compile_commands.json` so the active Debug or Release preset
updates that file after configuration. For command-line configuration, copy
`cmake_build/<preset>/compile_commands.json` to the repository root. Restart
`cmake_build/<preset>/compile_commands.json` to `src/Nitload.Native`. Restart
clangd after changing the database or renaming headers.
## Code Layout
src/Nitload.Native/nfn-src/thread.linux.c
+76 -8
View File
@@ -26,9 +26,18 @@ typedef struct nfn_linux_max_context {
int max_value;
} nfn_linux_max_context;
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c
typedef struct nfn_linux_fill_cpu_context {
uint64_t *internal;
uint32_t *global;
=======
typedef struct nif_linux_fill_cpu_context {
uint32_t *processor_group_ids;
uint32_t *socket_ids;
uint32_t *cpu_indices;
uint32_t *numa_node_ids;
int numa_id;
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c
int index;
} nfn_linux_fill_cpu_context;
@@ -118,14 +127,43 @@ static int nfn_linux_track_max_id(int value, void *context)
return 1;
}
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c
static int nfn_linux_fill_cpu_id(int value, void *context)
{
nfn_linux_fill_cpu_context *fill_context = (nfn_linux_fill_cpu_context *)context;
=======
static int nif_linux_read_socket_id(int cpu_id, uint32_t *socket_id)
{
char path[160];
char buffer[64];
char *end = NULL;
long parsed_value;
snprintf(path, sizeof(path),
"/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu_id);
if (!nif_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
if (fill_context->internal != NULL)
fill_context->internal[fill_context->index] = (uint64_t)fill_context->index;
if (fill_context->global != NULL)
fill_context->global[fill_context->index] = (uint32_t)value;
errno = 0;
parsed_value = strtol(buffer, &end, 10);
if (end == buffer || errno == ERANGE || parsed_value < 0 || parsed_value > UINT32_MAX)
return 0;
*socket_id = (uint32_t)parsed_value;
return 1;
}
static int nif_linux_fill_cpu_id(int value, void *context)
{
nif_linux_fill_cpu_context *fill_context = (nif_linux_fill_cpu_context *)context;
uint32_t socket_id = 0;
if (!nif_linux_read_socket_id(value, &socket_id))
return 0;
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c
fill_context->processor_group_ids[fill_context->index] = 0u;
fill_context->socket_ids[fill_context->index] = socket_id;
fill_context->cpu_indices[fill_context->index] = (uint32_t)value;
fill_context->numa_node_ids[fill_context->index] = (uint32_t)fill_context->numa_id;
fill_context->index++;
return 1;
@@ -211,12 +249,21 @@ static int nfn_linux_populate_writable_mapping(void *address, size_t size)
#endif
}
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id)
=======
/* Linux has no Windows-style processor groups; processor_group_id is ignored. */
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id)
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c
{
int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN);
(void)processor_group_id;
int cpu_count = (int)sysconf(_SC_NPROCESSORS_CONF);
if (cpu_index >= cpu_count)
cpu_count = cpu_index + 1;
if (cpu_count <= 0)
return;
@@ -227,11 +274,11 @@ NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id)
CPU_ZERO_S(size, set);
if (core_id < 0) {
if (cpu_index < 0) {
for (int i = 0; i < cpu_count; i++)
CPU_SET_S((int)i, size, set);
} else {
CPU_SET_S(core_id, size, set);
CPU_SET_S(cpu_index, size, set);
}
pthread_setaffinity_np(pthread_self(), size, set);
@@ -264,9 +311,18 @@ NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_eac
return 1;
}
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
=======
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
uint32_t *processor_group_ids,
uint32_t *socket_ids,
uint32_t *cpu_indices,
uint32_t *numa_node_ids)
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c
{
if (numa_id < 0)
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
cpu_indices == NULL || numa_node_ids == NULL)
return 0;
char path[128];
@@ -275,8 +331,20 @@ NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t
if (!nfn_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.linux.c
nfn_linux_fill_cpu_context fill_context = { internal, global, 0 };
return nfn_linux_visit_id_list(buffer, nfn_linux_fill_cpu_id, &fill_context);
=======
nif_linux_fill_cpu_context fill_context = {
processor_group_ids,
socket_ids,
cpu_indices,
numa_node_ids,
numa_id,
0
};
return nif_linux_visit_id_list(buffer, nif_linux_fill_cpu_id, &fill_context);
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.linux.c
}
NFN_API void *NFN_DECL nfn_numa_alloc(size_t size, int numa_id)
src/Nitload.Native/nfn-src/thread.windows.c
+141 -31
View File
@@ -3,33 +3,109 @@
#if NFN_IS_OS(NFN_OS_WINDOWS)
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.windows.c
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
NFN_API void NFN_DECL nfn_set_thread_affinity(int core_id, int numa_id)
=======
static int nif_windows_get_socket_map(uint32_t **out_socket_map, DWORD *out_group_count)
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.windows.c
{
if (core_id < 0 || numa_id < 0) {
/* Query which group the thread currently belongs to */
DWORD group_count = GetActiveProcessorGroupCount();
DWORD buffer_size = 0;
uint32_t *socket_map;
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *information;
BYTE *cursor;
uint32_t socket_id = 0;
if (group_count == 0)
return 0;
socket_map = (uint32_t *)malloc((size_t)group_count * 64u * sizeof(uint32_t));
if (socket_map == NULL)
return 0;
for (size_t i = 0; i < (size_t)group_count * 64u; i++)
socket_map[i] = UINT32_MAX;
GetLogicalProcessorInformationEx(RelationProcessorPackage, NULL, &buffer_size);
if (GetLastError() != ERROR_INSUFFICIENT_BUFFER || buffer_size == 0) {
free(socket_map);
return 0;
}
information = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)malloc(buffer_size);
if (information == NULL ||
!GetLogicalProcessorInformationEx(RelationProcessorPackage, information, &buffer_size)) {
free(information);
free(socket_map);
return 0;
}
cursor = (BYTE *)information;
while (cursor < (BYTE *)information + buffer_size) {
SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *entry =
(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)cursor;
if (entry->Relationship == RelationProcessorPackage) {
PROCESSOR_RELATIONSHIP *package = &entry->Processor;
for (WORD i = 0; i < package->GroupCount; i++) {
WORD group = package->GroupMask[i].Group;
KAFFINITY mask = package->GroupMask[i].Mask;
if (group >= group_count)
continue;
for (DWORD cpu = 0; cpu < 64; cpu++) {
if ((mask & ((KAFFINITY)1 << cpu)) != 0)
socket_map[(size_t)group * 64u + cpu] = socket_id;
}
}
socket_id++;
}
if (entry->Size == 0)
break;
cursor += entry->Size;
}
free(information);
*out_socket_map = socket_map;
*out_group_count = group_count;
return 1;
}
static int nif_windows_get_processor_node(WORD group, BYTE cpu, USHORT *node_id)
{
PROCESSOR_NUMBER processor;
memset(&processor, 0, sizeof(processor));
processor.Group = group;
processor.Number = cpu;
return GetNumaProcessorNodeEx(&processor, node_id) != 0;
}
/* Pin the calling thread using its processor group and group-local CPU index. */
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id)
{
if (cpu_index < 0 || processor_group_id < 0) {
GROUP_AFFINITY current;
GetThreadGroupAffinity(GetCurrentThread(), &current);
/* Build a full mask for that group's actual core count */
DWORD count = GetActiveProcessorCount(current.Group);
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = current.Group;
ga.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
GROUP_AFFINITY affinity;
memset(&affinity, 0, sizeof(affinity));
affinity.Group = current.Group;
affinity.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL);
return;
}
int index = core_id % 64;
DWORD count = GetActiveProcessorCount(numa_id);
if ((DWORD)index >= count)
if ((DWORD)processor_group_id >= GetActiveProcessorGroupCount() || cpu_index >= 64)
return;
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = (WORD)numa_id;
ga.Mask = (KAFFINITY)1 << index;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
USHORT node_id = 0;
if (!nif_windows_get_processor_node((WORD)processor_group_id, (BYTE)cpu_index, &node_id))
return;
GROUP_AFFINITY affinity;
memset(&affinity, 0, sizeof(affinity));
affinity.Group = (WORD)processor_group_id;
affinity.Mask = (KAFFINITY)1 << cpu_index;
SetThreadGroupAffinity(GetCurrentThread(), &affinity, NULL);
}
NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
@@ -39,30 +115,64 @@ NFN_API int NFN_DECL nfn_get_numa_overall(int *out_numa_count, int *count_of_eac
return 0;
*out_numa_count = (int)highest + 1;
for (UCHAR n = 0; n <= (UCHAR)highest; n++) {
ULONGLONG mask = 0;
GetNumaNodeProcessorMask(n, &mask);
count_of_each_numa[n] = (int)__popcnt64(mask);
if (count_of_each_numa != NULL) {
for (ULONG n = 0; n <= highest; n++)
count_of_each_numa[n] = 0;
}
DWORD group_count = GetActiveProcessorGroupCount();
for (DWORD group = 0; group < group_count; group++) {
for (DWORD cpu = 0; cpu < 64; cpu++) {
USHORT node_id = 0;
if (nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) &&
count_of_each_numa != NULL && node_id <= highest)
count_of_each_numa[node_id]++;
}
}
return 1;
}
<<<<<<< HEAD:src/Nitload.Native/nfn-src/thread.windows.c
NFN_API int NFN_DECL nfn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
=======
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
uint32_t *processor_group_ids,
uint32_t *socket_ids,
uint32_t *cpu_indices,
uint32_t *numa_node_ids)
>>>>>>> 3a2b1234782f893f5b71655289b372ffed8dfd93:src/Nitload.Native/nif-src/thread.windows.c
{
ULONGLONG mask = 0;
if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask))
if (numa_id < 0 || processor_group_ids == NULL || socket_ids == NULL ||
cpu_indices == NULL || numa_node_ids == NULL)
return 0;
int intra = 0;
while (mask) {
DWORD idx;
_BitScanForward64(&idx, mask);
if (internal != NULL)
internal[intra] = (uint64_t)intra;
global[intra] = (uint32_t)idx;
intra++;
mask &= mask - 1;
uint32_t *socket_map = NULL;
DWORD group_count = 0;
if (!nif_windows_get_socket_map(&socket_map, &group_count))
return 0;
int index = 0;
for (DWORD group = 0; group < group_count; group++) {
for (DWORD cpu = 0; cpu < 64; cpu++) {
USHORT node_id = 0;
if (!nif_windows_get_processor_node((WORD)group, (BYTE)cpu, &node_id) ||
node_id != (USHORT)numa_id)
continue;
uint32_t socket_id = socket_map[(size_t)group * 64u + cpu];
if (socket_id == UINT32_MAX) {
free(socket_map);
return 0;
}
processor_group_ids[index] = (uint32_t)group;
socket_ids[index] = socket_id;
cpu_indices[index] = cpu;
numa_node_ids[index] = node_id;
index++;
}
}
free(socket_map);
return 1;
}
src/Nitload.Native/nif-head/NitloadFrameWork.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef NIF_NATIVE_H
#define NIF_NATIVE_H
#ifdef __cplusplus
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
#endif
#include "nif_internal.h"
#include "nif_target.h"
#include <immintrin.h>
#include <assert.h>
#include <glfw_function.h>
#define byte char
#define cs_char uint16_t
#define nif_null_assert(ptr) assert(ptr != NULL)
#ifndef NIF_
#endif
NIF_API void NIF_DECL nif_load_glfw_functions(GLFWFunctionTable *table);
NIF_API void NIF_DECL nif_load_intrinsic_function(void *intrinsic_context, uint64_t id);
NIF_API SIMDID NIF_DECL nif_get_simd_level_global();
/* Per-core variant: always re-executes CPUID, never caches.
* Must be called on the target core (after affinity is set). */
NIF_API SIMDID NIF_DECL nif_get_simd_level_local();
NIF_API SIMDID NIF_DECL nif_get_unite_simd(SIMDID left, SIMDID right);
NIF_API int NIF_DECL nif_compare_simd(SIMDID left, SIMDID right);
NIF_API void NIF_DECL nif_set_thread_affinity(int cpu_index, int processor_group_id);
NIF_API void* NIF_DECL nif_numa_alloc(size_t size, int numa_id);
NIF_API void NIF_DECL nif_numa_free(void *ptr, size_t size, int numa_id);
NIF_API int NIF_DECL nif_get_numa_each(int numa_id,
uint32_t *processor_group_ids,
uint32_t *socket_ids,
uint32_t *cpu_indices,
uint32_t *numa_node_ids);
NIF_API int NIF_DECL nif_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
#endif
test/STgSharpTest/UnitTest1.cs
+2 -2
View File
@@ -7,7 +7,7 @@
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation
// files (the ¡°Software¡±), to deal in the Software without restriction,
// files (the ��Software��), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
@@ -17,7 +17,7 @@
// this permission notice shall be included in all copies
// or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED ¡°AS IS¡±,
// THE SOFTWARE IS PROVIDED ��AS IS��,
// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
test/StgSharpDebug/TestSwiss.cs
-46
View File
@@ -1,46 +0,0 @@
// -----------------------------------------------------------------------------
// file="TestSwiss"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using Nitload.Common.Collections;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
namespace Nitload
{
internal static partial class Program
{
public static void TestSwiss()
{
SwissTable swiss = SwissTable.Create();
for (nuint i = 0; i < 5000; i++)
{
Console.Write($"Set {i}: ");
_ = swiss.TryAddOrSet(i, i * 2, out _);
Console.WriteLine($"Set a kvp of ({i},{i * 2})");
}
Random rand = new Random();
for (int i = 0; i < 100; i++)
{
Console.Write($"Get {i}: ");
nuint key = (nuint)rand.Next(4000, 5000);
if (swiss.TryGet(key, out nuint value))
{
Console.WriteLine($"Success in get a kvp of ({key},{value})");
} else
{
Console.WriteLine($"Failed to get a kvp of ({key},?)");
}
}
}
}
}