Using new file structure. Use new regexir display method. Fix line dependency error in prefix analysing.

Signed-off-by: Nitload <YoumuLiu_NSI@whv0t.onmicrosoft.com>
This commit is contained in:
Nitload
2026-07-12 02:04:57 +08:00
parent ae8a09efc2
commit b6cd2fc256
476 changed files with 16440 additions and 502 deletions
@@ -0,0 +1,204 @@
// -----------------------------------------------------------------------------
// file="BidirectionalDictionary"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
namespace StgSharp.Collections
{
public sealed class BidirectionalDictionary<TFirst, TSecond> : IBidirectionalDictionary<TFirst, TSecond>
where TFirst : notnull
where TSecond : notnull
{
private readonly Dictionary<TFirst, TSecond> _forward;
private readonly Dictionary<TSecond, TFirst> _reverse;
public BidirectionalDictionary()
{
_forward = [];
_reverse = [];
}
public TFirst this[
TSecond key
]
{
get => _reverse.TryGetValue(key, out TFirst value) ?
value :
throw new ArgumentException($"Cannot find key {key} in dictionary.");
set
{
if (_reverse.TryGetValue(key, out TFirst origin))
{
_reverse[key] = value;
_ = _forward.Remove(origin);
} else
{
_reverse.Add(key, value);
}
_forward.Add(value, key);
}
}
public TSecond this[
TFirst key
]
{
get => _forward.TryGetValue(key, out TSecond value) ?
value :
throw new ArgumentException($"Cannot find key {key} in dictionary.");
set
{
if (_forward.TryGetValue(key, out TSecond origin))
{
_forward[key] = value;
_ = _reverse.Remove(origin);
} else
{
_forward.Add(key, value);
}
_reverse.Add(value, key);
}
}
public bool IsReadOnly => false;
public ICollection<TFirst> FirstIndex => _forward.Keys;
public ICollection<TSecond> SecondIndex => _reverse.Keys;
public int Count => _forward.Count;
public IReadOnlyDictionary<TFirst, TSecond> Forward => new ReadOnlyDictionary<TFirst, TSecond>(
_forward);
public IReadOnlyDictionary<TSecond, TFirst> Reverse => new ReadOnlyDictionary<TSecond, TFirst>(
_reverse);
public void Add(
KeyValuePair<TFirst, TSecond> item
)
{
if (_forward.ContainsKey(item.Key) || _reverse.ContainsKey(item.Value)) {
throw new ArgumentException("Duplicate key or value.");
}
_forward.Add(item.Key, item.Value);
_reverse.Add(item.Value, item.Key);
}
public void Add(
TFirst key,
TSecond value
)
{
if (_forward.ContainsKey(key) || _reverse.ContainsKey(value)) {
throw new ArgumentException("Duplicate key or value.");
}
_forward.Add(key, value);
_reverse.Add(value, key);
}
public void Clear()
{
_forward.Clear();
_reverse.Clear();
}
public bool Contains(
KeyValuePair<TFirst, TSecond> item
)
{
return _forward.Contains(item) &&
_reverse.TryGetValue(item.Value, out TFirst var) &&
var!.Equals(item.Key);
}
public bool Contains(
TSecond key
)
{
return _reverse.ContainsKey(key);
}
public bool Contains(
TFirst key
)
{
return _forward.ContainsKey(key);
}
public void CopyTo(
KeyValuePair<TFirst, TSecond>[] array,
int arrayIndex
)
{
if (array == null) {
throw new ArgumentNullException(nameof(array));
}
if (arrayIndex < 0 || arrayIndex > array.Length) {
throw new ArgumentOutOfRangeException(nameof(arrayIndex));
}
if (array.Length - arrayIndex < _forward.Count) {
throw new ArgumentException("The array is too small to copy the elements.");
}
foreach (KeyValuePair<TFirst, TSecond> pair in _forward) {
array[arrayIndex++] = pair;
}
}
public IEnumerator<KeyValuePair<TFirst, TSecond>> GetEnumerator()
{
foreach (KeyValuePair<TFirst, TSecond> pair in _forward) {
yield return pair;
}
}
public bool Remove(
TFirst key
)
{
throw new NotImplementedException();
}
public bool Remove(
KeyValuePair<TFirst, TSecond> item
)
{
if (_forward.Remove(item.Key)) {
return _reverse.Remove(item.Value);
}
return false;
}
public bool TryGetValue(
TFirst key,
out TSecond value
)
{
return _forward.TryGetValue(key, out value);
}
public bool TryGetValue(
TSecond key,
out TFirst value
)
{
return _reverse.TryGetValue(key, out value);
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
@@ -0,0 +1,80 @@
// -----------------------------------------------------------------------------
// file="CapacityFixedBag"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Reflection;
namespace StgSharp.Collections
{
public sealed class CapacityFixedBag<T> : IEnumerable<T>
{
private readonly T[] _buffer;
public CapacityFixedBag(
int capacity
)
{
_buffer = new T[capacity];
}
public int Count { get; set; }
public int Capacity => _buffer.Length;
public void Add(
T item
)
{
if (Count >= _buffer.Length) {
throw new InvalidOperationException("Bag capacity exceeded.");
}
_buffer[Count] = item;
Count++;
}
public IEnumerator<T> GetEnumerator()
{
for (int i = 0; i < Count; i++) {
yield return _buffer[i];
}
}
public void Remove(
T item
)
{
int index = Array.IndexOf(_buffer, item, 0, Count);
if (index >= 0)
{
_buffer[index] = _buffer[Count - 1];
_buffer[Count - 1] = default!;
Count--;
}
}
public void RemoveAt(
int index
)
{
if (index >= 0)
{
_buffer[index] = _buffer[Count - 1];
_buffer[Count - 1] = default!;
Count--;
}
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
@@ -0,0 +1,203 @@
// -----------------------------------------------------------------------------
// file="CapacityFixedList"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace StgSharp.Collections
{
public static class CapacityFixedListBuilder
{
internal static CapacityFixedList<T> Create<T>(
ReadOnlySpan<T> values
)
{
return new CapacityFixedList<T>(values);
}
}
[CollectionBuilder(typeof(CapacityFixedListBuilder), nameof(CapacityFixedListBuilder.Create))]
public class CapacityFixedList<T> : IList<T>
{
private T[] _values;
private readonly int _capacity;
private int _index;
public CapacityFixedList(
int size
)
{
_values = new T[size];
_capacity = size;
}
public CapacityFixedList(
ReadOnlySpan<T> values
)
{
_values = values.ToArray();
}
public T this[
int index
]
{
get => index < _index ?
_values[index] :
throw new ArgumentOutOfRangeException(nameof(index));
set => _values[index] =
index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
}
public bool IsReadOnly => false;
public int Count => _index + 1;
public void Add(
T item
)
{
if (_index < _capacity - 1)
{
_index++;
this[_index] = item;
}
}
public void Clear()
{
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
Array.Fill(_values, default);
}
_index = 0;
}
public bool Contains(
T item
)
{
if (item is null) {
return false;
}
for (int i = 0; i <= _index; i++)
{
if (item.Equals(_values[i])) {
return true;
}
}
return false;
}
public void CopyTo(
T[] array,
int arrayIndex
)
{
_values.CopyTo(array, arrayIndex);
}
public IEnumerator<T> GetEnumerator()
{
for (int i = 0; i <= _index; i++) {
yield return _values[i];
}
}
public int IndexOf(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
{
if ((v = _values[i]) != null && v.Equals(item)) {
return i;
}
}
return -1;
}
public bool Remove(
T item
)
{
T v;
for (int i = 0; i <= _index; i++)
{
if ((v = _values[i]) != null && v.Equals(item))
{
_values[i] = _values[_index];
_index--;
return true;
}
}
return false;
}
public void RemoveAll(
Predicate<T> predicate
)
{
if (predicate is null) {
return;
}
for (int i = _index; i > 0; i--)
{
if (predicate(_values[i]))
{
_values[i] = _values[_index];
_index--;
}
}
}
public void RemoveAt(
int index
)
{
if (index <= _index)
{
Array.Copy(_values, index + 1, _values, index, _index - index);
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
_values[_index] = default!;
}
_index--;
}
}
public void Resize(
int newSize
)
{
Array.Resize(ref _values, newSize);
}
internal Span<T> ToSpan()
{
return _values;
}
IEnumerator IEnumerable.GetEnumerator()
{
yield return GetEnumerator();
}
void IList<T>.Insert(
int index,
T item
)
{
throw new NotSupportedException();
}
}//------------------------------------- End of Class ------------------------------------------
}
@@ -0,0 +1,166 @@
// -----------------------------------------------------------------------------
// file="CapacityFixedStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
namespace StgSharp.Collections
{
[CollectionBuilder(typeof(CapacityFixedStackBuilder), nameof(CapacityFixedStackBuilder.Create))]
public class CapacityFixedStack<T> : IEnumerable<T>
{
private readonly T[] _values;
// _index points to the next free slot (top empty position). Therefore Count == _index.
private int _index;
public CapacityFixedStack(
int size
)
{
ArgumentOutOfRangeException.ThrowIfNegative(size);
_values = new T[size];
_index = 0; // empty stack
}
public T? First => Volatile.Read(ref _index) > 0 ? _values[0] : default;
public bool IsEmpty => Volatile.Read(ref _index) == 0;
public int Count => Volatile.Read(ref _index);
public int Capacity => _values.Length;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ReadOnlySpan<T> AsSpan()
{
return _values.AsSpan(0, _index);
}
public void Clear()
{
if (_index > 0)
{
Array.Clear(_values, 0, _index);
_index = 0;
}
}
public IEnumerator<T> GetEnumerator()
{
for (int i = _index - 1; i >= 0; i--) {
yield return _values[i];
}
}
public T Peek()
{
if (_index == 0) {
throw new InvalidOperationException("Stack is empty.");
}
return _values[_index - 1];
}
public T Pop()
{
if (_index == 0) {
throw new InvalidOperationException("Stack is empty.");
}
_index--;
T item = _values[_index];
_values[_index] = default!; // Clear reference for GC
return item;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T[] PopRange(
int count
)
{
if (count <= 0 || _index == 0) {
return Array.Empty<T>();
}
int n = Math.Min(count, _index);
int start = _index - n;
T[] ret = new T[n];
Array.Copy(_values, start, ret, 0, n);
Array.Clear(_values, start, n);
_index -= n;
return ret;
}
public void Push(
T item
)
{
if (_index >= _values.Length) {
throw new InvalidOperationException("Stack is full.");
}
_values[_index] = item;
_index++;
}
public void PushRange(
ReadOnlySpan<T> span
)
{
if (span.Length == 0) {
return;
}
if (_index + span.Length > _values.Length) {
throw new InvalidOperationException("Not enough space in the stack to push the entire span.");
}
span.CopyTo(_values.AsSpan(_index, span.Length));
_index += span.Length;
}
public bool TryPop(
out T value
)
{
if (_index == 0)
{
value = default!;
return false;
}
_index--;
value = _values[_index];
_values[_index] = default!;
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal Span<T> GetValueLayout()
{
return _values.AsSpan();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
public static class CapacityFixedStackBuilder
{
public static CapacityFixedStack<T> Create<T>(
ReadOnlySpan<T> values
)
{
CapacityFixedStack<T> stack = new(values.Length);
stack.PushRange(values); // Push in order
return stack;
}
}
}
@@ -0,0 +1,284 @@
// -----------------------------------------------------------------------------
// file="ConcurrentCapacityFixedStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading;
namespace StgSharp.Collections
{
/// <summary>
/// Lock-free, fixed-capacity stack implemented on top of an array using atomic index
/// operations. - No linked list; only array slots - Uses CAS on a shared top index to
/// reserve/release slots - Per-slot state flag to safely publish/consume values under races
/// </summary>
public class ConcurrentCapacityFixedStack<T> : IEnumerable<T>
{
private readonly int[] _states; // 0 = empty, 1 = full
private readonly T[] _values;
private readonly int _capacity;
// Number of full slots (observable count)
private int _count;
// Next push position (0.._capacity). Only modified via CAS.
private int _top;
public ConcurrentCapacityFixedStack(
int capacity
)
{
if (capacity <= 0) {
throw new ArgumentOutOfRangeException(nameof(capacity));
}
_capacity = capacity;
_values = new T[capacity];
_states = new int[capacity];
_top = 0;
_count = 0;
}
public bool IsEmpty => Count == 0;
public int Capacity => _capacity;
public int Count => Volatile.Read(ref _count);
public void Clear()
{
// Pop all elements to return slots to empty state
while (TryPop(out _)) { }
}
public IEnumerator<T> GetEnumerator()
{
T[] snap = ToArraySnapshot();
for (int i = 0; i < snap.Length; i++) {
yield return snap[i];
}
}
public T Peek()
{
if (TryPeek(out T? v)) {
return v;
}
throw new InvalidOperationException("Stack is empty.");
}
public T Pop()
{
if (TryPop(out T value)) {
return value;
}
throw new InvalidOperationException("Stack is empty.");
}
public T[] PopRange(
int count
)
{
if (count <= 0) {
return [];
}
T[] result = new T[count];
int i = 0;
for (; i < count; i++)
{
if (!TryPop(out result[i]))
{
break;
}
}
if (i == count) {
return result;
}
if (i == 0) {
return [];
}
T[] trimmed = new T[i];
Array.Copy(result, trimmed, i);
return trimmed;
}
public void Push(
T item
)
{
SpinWait sw = default;
while (true)
{
int oldTop = Volatile.Read(ref _top);
if (oldTop >= _capacity) {
throw new InvalidOperationException("Stack is full.");
}
if (Interlocked.CompareExchange(ref _top, oldTop + 1, oldTop) != oldTop)
{
sw.SpinOnce();
continue; // failed to reserve slot
}
// We exclusively own slot oldTop. Publish value then state.
_values[oldTop] = item;
Volatile.Write(ref _states[oldTop], 1); // publish
_ = Interlocked.Increment(ref _count);
return;
}
}
public void PushRange(
ReadOnlySpan<T> span
)
{
for (int i = 0; i < span.Length; i++) {
Push(span[i]);
}
}
/// <summary>
/// Best-effort snapshot for enumeration; not strongly consistent under concurrent
/// mutations.
/// </summary>
public T[] ToArraySnapshot()
{
int expected = Math.Min(Volatile.Read(ref _count), _capacity);
if (expected == 0) {
return Array.Empty<T>();
}
T[] snap = new T[expected];
int copied = 0;
int idx = Volatile.Read(ref _top) - 1;
while (idx >= 0 && copied < expected)
{
if (Volatile.Read(ref _states[idx]) == 1) {
snap[copied++] = _values[idx];
}
idx--;
}
if (copied == snap.Length) {
return snap;
}
T[] trimmed = new T[copied];
Array.Copy(snap, trimmed, copied);
return trimmed;
}
public bool TryPeek(
out T value
)
{
value = default!;
SpinWait sw = default;
while (true)
{
int top = Volatile.Read(ref _top);
if (top <= 0) {
return false;
}
int idx = top - 1;
// If not yet published or racing, retry/spin
if (Volatile.Read(ref _states[idx]) == 0)
{
if (Volatile.Read(ref _top) != top)
{
continue; // head moved, retry
}
sw.SpinOnce();
continue;
}
T v = _values[idx];
// Validate consistency
if (Volatile.Read(ref _top) == top && Volatile.Read(ref _states[idx]) == 1)
{
value = v;
return true;
}
}
}
public bool TryPop(
out T value
)
{
value = default!;
SpinWait sw = default;
while (true)
{
int oldTop = Volatile.Read(ref _top);
if (oldTop <= 0)
{
return false; // empty
}
int newTop = oldTop - 1; // index to consume
if (Interlocked.CompareExchange(ref _top, newTop, oldTop) != oldTop)
{
sw.SpinOnce();
continue; // failed to reserve element
}
// If producer hasn't published yet, roll back our reservation and retry.
if (Volatile.Read(ref _states[newTop]) == 0)
{
// attempt to restore top; ok if it fails due to concurrent operations
_ = Interlocked.CompareExchange(ref _top, newTop + 1, newTop);
sw.SpinOnce();
continue;
}
value = _values[newTop];
_values[newTop] = default!; // avoid ref retention
Volatile.Write(ref _states[newTop], 0); // mark empty
_ = Interlocked.Decrement(ref _count);
return true;
}
}
public int TryPopRange(
Span<T> destination
)
{
if (destination.IsEmpty) {
return 0;
}
int i = 0;
for (; i < destination.Length; i++)
{
if (!TryPop(out destination[i]))
{
break;
}
}
return i;
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
@@ -0,0 +1,98 @@
// -----------------------------------------------------------------------------
// file="ConcurrentFlexibleArray"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Collections
{
public class ConcurrentFlexibleArray<T>(
int capacity = 4
)
{
private volatile T[] _array = new T[capacity];
private readonly ReaderWriterLockSlim _rwLock = new();
public T this[
int index
]
{
get
{
_rwLock.EnterReadLock();
try
{
if ((uint)index >= (uint)_array.Length) {
throw new ArgumentOutOfRangeException(nameof(index));
}
return _array[index];
}
finally
{
_rwLock.ExitReadLock();
}
}
set
{
if ((uint)index >= (uint)_array.Length)
{
_rwLock.EnterWriteLock();
try
{
if ((uint)index >= (uint)_array.Length)
{
int newSize = Math.Max(_array.Length * 2, index + 1);
T[] newArray = new T[newSize];
Array.Copy(_array, newArray, _array.Length);
_array = newArray;
_array[index] = value;
}
}
finally
{
_rwLock.ExitWriteLock();
}
} else
{
_rwLock.EnterReadLock();
try
{
_array[index] = value;
}
finally
{
_rwLock.ExitReadLock();
}
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal Span<T> AsSpan()
{
return _array.AsSpan();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void ReleaseSpanOperation()
{
_rwLock.ExitReadLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void RequestSpanOperation()
{
_rwLock.EnterReadLock();
}
}
}
@@ -0,0 +1,75 @@
// -----------------------------------------------------------------------------
// file="IBidirectionalDictionary"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Collections.Generic;
namespace StgSharp.Collections
{
public interface IBidirectionalDictionary<TFirst, TSecond> : IDictionary<TFirst, TSecond>
{
TFirst this[
TSecond key
] { get; set; }
ICollection<TFirst> FirstIndex { get; }
ICollection<TSecond> SecondIndex { get; }
IReadOnlyDictionary<TFirst, TSecond> Forward { get; }
IReadOnlyDictionary<TSecond, TFirst> Reverse { get; }
/// <summary>
/// Determines whether the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains the
/// specified key.
/// </summary>
/// <param _label="key">
/// The value to locate in the <see cref="IBidirectionalDictionary{TFirst, TSecond}" />.
/// </param>
/// <returns>
/// true if the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains an element with
/// the specified key; otherwise, false.
/// </returns>
bool Contains(
TSecond key
);
/// <summary>
/// Determines whether the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains the
/// specified key.
/// </summary>
/// <param _label="key">
/// The value to locate in the <see cref="IBidirectionalDictionary{TFirst, TSecond}" />.
/// </param>
/// <returns>
/// true if the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains an element with
/// the specified key; otherwise, false.
/// </returns>
bool Contains(
TFirst key
);
bool TryGetValue(
TSecond key,
out TFirst value
);
#pragma warning disable CA1033
ICollection<TFirst> IDictionary<TFirst, TSecond>.Keys => FirstIndex;
ICollection<TSecond> IDictionary<TFirst, TSecond>.Values => SecondIndex;
bool IDictionary<TFirst, TSecond>.ContainsKey(
TFirst key
)
{
return Contains(key);
}
#pragma warning restore CA1033
}
}
@@ -0,0 +1,193 @@
// -----------------------------------------------------------------------------
// file="LinearBag"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace StgSharp.Collections
{
public static class LinearBagBuilder
{
/// <summary>
/// Build a <see cref="LinearBag{T}" /> from a read-only span. Capacity will exactly fit
/// the input.
/// </summary>
public static LinearBag<T> Create<T>(
ReadOnlySpan<T> values
)
{
if (values.IsEmpty) {
return new LinearBag<T>(4);
}
LinearBag<T> bag = new(values.Length);
for (int i = 0; i < values.Length; i++) {
bag.Add(values[i]);
}
return bag;
}
}
/// <summary>
/// A very lightweight contiguous storage structure: unordered, not enumerable by design.
/// Provides O(1) Add and O(1) RemoveAt (by swapping with the tail). Intended for high-
/// performance scenarios where element ordering does not matter. NOT thread-safe.
/// </summary>
/// <typeparam name="T">
/// Element type.
/// </typeparam>
[CollectionBuilder(typeof(LinearBagBuilder), nameof(LinearBagBuilder.Create))]
public class LinearBag<T>(
int capacity = 4
)
{
// Invariant: valid elements occupy indexes [0, Count-1]. Count is the number of live elements.
private T[] _items = new T[Math.Max(0, capacity)];
/// <summary>
/// Access element by index (0..Count-1). Throws if out of range.
/// </summary>
public T this[
int index
]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count);
return _items[index];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
set
{
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count);
_items[index] = value;
}
}
/// <summary>
/// Total allocated slots.
/// </summary>
public int Capacity => _items.Length;
/// <summary>
/// Number of currently stored elements (valid index range 0..Count-1).
/// </summary>
public int Count { get; private set; }
/// <summary>
/// Adds an item (unordered). Expands internal array if needed.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Add(
T item
)
{
if (Count == _items.Length) {
EnsureCapacity(Count + 1);
}
_items[Count] = item;
Count++;
}
/// <summary>
/// Returns a span over the live portion of the bag (0..Count-1).
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Span<T> AsSpan()
{
return new Span<T>(_items, 0, Count);
}
/// <summary>
/// Clears all elements and zeroes out references to avoid retention. Count becomes 0.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
Array.Clear(_items, 0, Count);
Count = 0;
}
/// <summary>
/// Fast clear: only resets Count (leaves underlying array unchanged). References are NOT
/// released. Use only when you are certain old references can remain.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void FastClear()
{
Count = 0;
}
/// <summary>
/// Removes the first occurrence of value (unordered). Does nothing if not found.
/// </summary>
public void Remove(
T value,
EqualityComparer<T> comparer = default
)
{
comparer ??= EqualityComparer<T>.Default;
for (int i = 0; i < Count; i++)
{
if (comparer.Equals(_items[i], value))
{
RemoveAt(i);
return;
}
}
}
/// <summary>
/// Removes element at index by swapping the last element into its slot (unless already
/// last). Preserves O(1) removal at cost of ordering.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void RemoveAt(
int index
)
{
if ((uint)index >= (uint)Count) {
throw new ArgumentOutOfRangeException(nameof(index));
}
int last = Count - 1;
if (index != last) {
_items[index] = _items[last];
}
_items[last] = default!; // release reference / reset value
Count = last;
}
/// <summary>
/// Ensures capacity is at least <paramref name="min" />. Doubles size when growing.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void EnsureCapacity(
int min
)
{
if (_items.Length >= min) {
return;
}
int newCapacity = (_items.Length == 0) ? 4 : (_items.Length * 2);
if (newCapacity < min) {
newCapacity = min;
}
Array.Resize(ref _items, newCapacity);
}
}
}
@@ -0,0 +1,128 @@
// -----------------------------------------------------------------------------
// file="ListSegment"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Serialization;
namespace StgSharp.Collections
{
/// <summary>
/// Part of a collection. This contains a direct reference to _origin collection, so all
/// change to _origin collection will be reflected in this object.
/// </summary>
/// <typeparam name="T">
///
/// </typeparam>
public struct ListSegment<T> : ICollection<T>
{
private int _begin;
private int _length;
private List<T> _collection;
public ListSegment(
List<T> collection,
int begin,
int length
)
{
ArgumentNullException.ThrowIfNull(collection);
ArgumentOutOfRangeException.ThrowIfLessThan(begin, 0);
ArgumentOutOfRangeException.ThrowIfLessThan(length, 0);
ArgumentOutOfRangeException.ThrowIfLessThan(begin + length, collection.Count);
_collection = collection;
_begin = begin;
_length = length;
}
public T this[
int indexValue
]
{
get
{
ArgumentOutOfRangeException.ThrowIfLessThan(indexValue, 0);
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(indexValue, _length);
return _collection[indexValue + _begin];
}
}
public T this[
Index i
]
{
get
{
int offset = i.IsFromEnd ? (_length - i.Value) : i.Value;
ArgumentOutOfRangeException.ThrowIfLessThan(offset, 0);
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, _length);
return _collection[offset + _begin];
}
}
public bool IsReadOnly => true;
public int Count => _length;
public bool Contains(
T item
)
{
int index = _collection.IndexOf(item);
return (index >= _begin) && (index < _length + index);
}
public void CopyTo(
T[] array,
int arrayIndex
)
{
ArgumentNullException.ThrowIfNull(array);
Span<T> span = CollectionsMarshal.AsSpan<T>(_collection).Slice(_begin, _length);
Span<T> target = new(array, arrayIndex, _length);
span.CopyTo(target);
}
public IEnumerator<T> GetEnumerator()
{
for (int i = _begin; i < _length; i++) {
yield return this[i];
}
}
void ICollection<T>.Add(
T item
)
{
throw new NotSupportedException();
}
void ICollection<T>.Clear()
{
throw new NotSupportedException();
}
bool ICollection<T>.Remove(
T item
)
{
throw new NotSupportedException();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
@@ -0,0 +1,169 @@
// -----------------------------------------------------------------------------
// file="RandomAccessibleStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
namespace StgSharp.Collections
{
public class RandomAccessibleStack<T> : IEnumerable<T>
{
private const int DefaultCapacity = 4;
private T[] _array;
private int _size;
private int _version;
/// <summary>
/// Initializes a new instance of the RandomAccessStack class that is empty and has the
/// default initial capacity.
/// </summary>
public RandomAccessibleStack()
{
_array = new T[DefaultCapacity];
}
/// <summary>
/// Initializes a new instance of the RandomAccessStack class that is empty and has the
/// specified initial capacity.
/// </summary>
public RandomAccessibleStack(
int capacity
)
{
if (capacity < 0) {
throw new ArgumentOutOfRangeException(
nameof(capacity), "Capacity must be non-negative.");
}
_array = new T[capacity];
}
/// <summary>
/// Gets or sets the element at the specified index in the stack (0 = bottom, Count-1 =
/// top).
/// </summary>
public T this[
int index
]
{
get
{
if (index < 0 || index >= _size) {
throw new ArgumentOutOfRangeException(nameof(index));
}
return _array[index];
}
/*
set
{
if( index < 0 || index >= PrimSize ) {
throw new ArgumentOutOfRangeException( nameof( index ) );
}
_array[ index ] = value;
_version++;
}
/**/
}
/// <summary>
/// Gets the number of elements contained in the stack.
/// </summary>
public int Count => _size;
/// <summary>
/// Removes all objects from the stack.
/// </summary>
public void Clear()
{
Array.Clear(_array, 0, _size);
_size = 0;
_version++;
}
/// <summary>
/// Returns an enumerator that iterates through the stack from top to bottom. Throws if
/// the stack is modified during iteration.
/// </summary>
public IEnumerator<T> GetEnumerator()
{
int expectedVersion = _version;
for (int i = _size - 1; i >= 0; i--)
{
if (expectedVersion != _version) {
throw new InvalidOperationException(
"Collection was modified during enumeration.");
}
yield return _array[i];
}
}
/// <summary>
/// Returns the object at the top of the stack without removing it.
/// </summary>
public T Peek()
{
if (_size == 0) {
throw new InvalidOperationException("Stack is empty.");
}
return _array[_size - 1];
}
/// <summary>
/// Removes and returns the object at the top of the stack.
/// </summary>
public T Pop()
{
if (_size == 0) {
throw new InvalidOperationException("Stack is empty.");
}
T item = _array[--_size];
_array[_size] = default!;
_version++;
return item;
}
/// <summary>
/// Adds an item to the top of the stack.
/// </summary>
public void Push(
T item
)
{
if (_size == _array.Length) {
EnsureCapacity(_size + 1);
}
_array[_size++] = item;
_version++;
}
private void EnsureCapacity(
int min
)
{
int newCapacity = _array.Length == 0 ? DefaultCapacity : _array.Length * 2;
if (newCapacity < min) {
newCapacity = min;
}
Array.Resize(ref _array, newCapacity);
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
@@ -0,0 +1,130 @@
// -----------------------------------------------------------------------------
// file="SwissTable.X86"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Numerics;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
namespace StgSharp.Collections
{
public partial class SwissTable
{
private unsafe class BucketScannerX86 : IBucketScanner
{
public bool TryFind(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector);
int mask = Sse2.MoveMask(cmpResult);
while (mask != 0)
{
int bitPos = BitOperations.TrailingZeroCount(mask);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
return true;
}
mask &= mask - 1;
}
value = default;
return false;
}
public bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control);
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
value = bucket->Values[index];
bucket->Control[index] = 0xFE; // mark as DELETED
// no need to remove Key/Value
return true;
}
matches &= matches - 1;
}
value = default;
return false;
}
public bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
)
{
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0)
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
Vector128<byte> bucketCtrl = Sse2.LoadVector128(
(byte*)&bucket->Control);
// === Phase 1: Look up an existing key ===
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
int matches = Sse2.MoveMask(matchMask);
while (matches != 0)
{
int bitPos = BitOperations.TrailingZeroCount(matches);
int index = bitPos;
if (bucket->Keys[index] == key)
{
// Key already exists -> update value
bucket->Values[index] = value;
isNewKey = false;
return true;
}
matches &= matches - 1;
}
// === Phase 2: Track an available insertion slot ===
int emptyOrDeleted = Sse2.MoveMask(bucketCtrl);
if (emptyOrDeleted == 0)
{
isNewKey = false;
return false; // bucket full
}
int slot = BitOperations.TrailingZeroCount(emptyOrDeleted);
bucket->Control[slot] = ctrl;
bucket->Keys[slot] = key;
bucket->Values[slot] = value;
isNewKey = true;
return true;
}
}
}
}
@@ -0,0 +1,224 @@
// -----------------------------------------------------------------------------
// file="SwissTable"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
namespace StgSharp.Collections
{
/// <summary>
///
/// </summary>
public sealed unsafe partial class SwissTable : IDisposable
{
private const int BucketCount = 1 << EntryMaskWidth;
private const int BucketSize = 16;
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<Entry>();
private readonly Entry* _entries;
private readonly Action<nuint> _freeHandle;
private bool _disposed;
private SwissTable(
Entry* buffer,
Action<nuint> freeHandle
)
{
_entries = buffer;
_freeHandle = freeHandle;
}
public static SwissTable Create()
{
Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf<Entry>(), 16);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle));
}
public static SwissTable Create(
Func<nuint, nuint> allocator,
Action<nuint> freeHandle
)
{
Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize);
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, freeHandle);
}
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_freeHandle((nuint)_entries);
GC.SuppressFinalize(this);
}
public bool TryAddOrSet(
nuint key,
nuint value,
out bool isNewKey
)
{
uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey);
}
public bool TryGet(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryFind(key, hash, bucket, out value);
/*
* hash = CRC32 of key
* entry = low 8 of hash
* ctrl = 8-15 of hash
* index = rest of hash
* find value of hash
*/
}
public bool TryRemove(
nuint key,
out nuint value
)
{
uint hash = BitOperations.Crc32C(CrcMagic, key);
ushort entry = (ushort)(hash & (BucketCount - 1));
Bucket* bucket = &(*_entries)[entry];
return IBucketScanner.Current.TryRemove(key, hash, bucket, out value);
}
internal static SwissTable Create(
nuint bufferPtr,
Action<nuint> FreeHandle
)
{
Entry* buffer = (Entry*)bufferPtr;
for (int i = 0; i < BucketCount; i++)
{
// Set all control bytes to 0xFF (empty)
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
}
return new SwissTable(buffer, FreeHandle);
}
~SwissTable()
{
Dispose();
}
[InlineArray(BucketCount)]
private struct Entry
{
public Bucket Bucket;
}
private unsafe struct Bucket
{
public BucketControl Control;
public BucketKeyStorage Keys;
public BucketValueStorage Values;
}
private interface IBucketScanner
{
static IBucketScanner Current { get; } = Create();
bool TryFind(
nuint key,
uint Hash,
Bucket* bucket,
out nuint value
);
unsafe bool TryRemove(
nuint key,
uint hash,
Bucket* bucket,
out nuint value
);
bool TrySet(
nuint key,
uint hash,
Bucket* bucket,
nuint value,
out bool isNewKey
);
private static IBucketScanner Create()
{
if (Sse42.IsSupported)
{
return new BucketScannerX86();
} else if (ArmBase.IsSupported)
{
throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform.");
} else
{
throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform.");
}
}
}
[InlineArray(BucketSize)]
private struct BucketControl
{
public byte Control;
}
[InlineArray(BucketSize)]
private struct BucketKeyStorage
{
public nuint value;
}
[InlineArray(BucketSize)]
private struct BucketValueStorage
{
public nuint value;
}
}
}
@@ -0,0 +1,156 @@
// -----------------------------------------------------------------------------
// file="UnmanagedArray"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Collections
{
/// <summary>
/// Zero-overhead view over a contiguous region of unmanaged memory. Does not own the memory —
/// caller is responsible for lifetime management.
/// </summary>
/// <typeparam name="T">
/// Element type. Must be unmanaged.
/// </typeparam>
public readonly unsafe struct UnmanagedArray<T> where T : unmanaged
{
private static readonly nint ElementSize = sizeof(T);
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public UnmanagedArray(
T* pointer,
int length
)
{
Address = pointer;
Length = length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public UnmanagedArray(
nuint address,
int length
)
{
Address = (T*)address;
Length = length;
}
/// <summary>
/// Indexed access. No bounds check in Release builds.
/// </summary>
public ref T this[
int index
]
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get
{
Debug.Assert((uint)index < (uint)Length,
"UnmanagedArray index out of range");
return ref Address[index];
}
}
/// <summary>
/// Raw pointer to the first element.
/// </summary>
public T* Address
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get;
init;
}
/// <summary>
/// Element count.
/// </summary>
public int Length
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get;
init;
}
/// <summary>
/// Total byte size of the array (for allocation calculations).
/// </summary>
public int ByteSize => Length * sizeof(T);
/// <summary>
/// Wraps the array as a <see cref="Span{T}" />.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Span<T> AsSpan()
{
return new Span<T>(Address, Length);
}
/// <summary>
/// Wraps a sub-range as a <see cref="Span{T}" />.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Span<T> AsSpan(
int start,
int length
)
{
Debug.Assert(((uint)start) + ((uint)length) <= (uint)Length,
"UnmanagedArray.AsSpan range out of bounds");
return new Span<T>(Address + start, length);
}
/// <summary>
/// Zero-fill all elements.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
Unsafe.InitBlock(Address, 0, (uint)(Length * sizeof(T)));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int IndexOf(
T* ptr
)
{
int offset = (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
ArgumentOutOfRangeException.ThrowIfNegative(offset, nameof(ptr));
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, Length, nameof(ptr));
return offset;
}
/// <summary>
/// Returns a pointer to element at <paramref name="index" />. Useful for feeding SIMD
/// load instructions (e.g. Sse2.LoadVector128).
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T* PtrAt(
int index
)
{
Debug.Assert((uint)index < (uint)Length,
"UnmanagedArray.PtrAt index out of range");
return Address + index;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int UncheckedIndexOf(
T* ptr
)
{
return (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
}
}
}
@@ -0,0 +1,85 @@
// -----------------------------------------------------------------------------
// file="UnmanagedStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Collections
{
public unsafe class UnmanagedStack<T>(
T* address,
int capacity,
Action<nuint> freeHandle
) : IDisposable where T : unmanaged
{
private T* _array = address;
private readonly Action<nuint> _freeHandle = freeHandle;
private readonly int _capacity = capacity;
private int _index;
public void Clear()
{
_index = 0;
}
public void Dispose()
{
if (_array != null)
{
_freeHandle((nuint)_array);
_array = null;
}
GC.SuppressFinalize(this);
}
public T Pop()
{
if (_index == 0) {
throw new InvalidOperationException("Stack is empty.");
}
return _array[--_index];
}
public void Push(
T value
)
{
_array[_index] = value;
_index++;
}
public bool TryPop(
out T value
)
{
if (_index == 0)
{
value = default;
return false;
}
value = _array[--_index];
return true;
}
public bool TryPush(
T value
)
{
if (_index >= _capacity) {
return false;
}
_array[_index] = value;
_index++;
return true;
}
}
}
+170
View File
@@ -0,0 +1,170 @@
// -----------------------------------------------------------------------------
// file="Dialogue"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Pipes;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
namespace StgSharp
{
public static class Dialogue
{
private static Process? _dialogueProcess;
private static AnonymousPipeServerStream _dialogueClientPipe,_dialogueServerPipe;
private static int _readAndWriteThreadId;
private static ProcessStartInfo _dialogueProcessInfo;
private static StreamReader _dr;
private static StreamWriter _dw;
public static bool NeedShowWhenStartup { get; set; }
public static void PostError(
Exception ex
)
{
CreateDialogueProcessIfNotExist();
_dw.WriteLine($"posterror {ex.Message}");
_dialogueProcess!.Exited -= DialogueProcessExitCallback;
}
public static void SetReadAndWriteThread(
Thread thread
)
{
_readAndWriteThreadId = thread.ManagedThreadId;
}
internal static void CreateDialogueProcessIfNotExist()
{
if (_dialogueProcess != null) {
return;
}
if (!File.Exists(_dialogueProcessInfo.FileName)) {
return;
}
_dialogueProcess = Process.Start(_dialogueProcessInfo);
_dr = new StreamReader(_dialogueClientPipe);
_dw = new StreamWriter(_dialogueServerPipe);
_dialogueProcess!.Exited += DialogueProcessExitCallback;
_dialogueServerPipe.DisposeLocalCopyOfClientHandle();
_dialogueClientPipe.DisposeLocalCopyOfClientHandle();
}
internal static void LoadDialogue()
{
_dialogueServerPipe = new AnonymousPipeServerStream(
PipeDirection.Out, HandleInheritability.Inheritable);
_dialogueClientPipe = new AnonymousPipeServerStream(
PipeDirection.In, HandleInheritability.Inheritable);
string route = Assembly.GetAssembly(typeof(World))?.Location ?? string.Empty;
string path = Path.GetDirectoryName(route) ?? string.Empty;
string exeName;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
exeName = ".exe";
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
exeName = string.Empty;
} else
{
throw new PlatformNotSupportedException("Unknown OS platform");
}
string consoleRoute = Path.Combine(
path, @"StgSharp.TerminalDialogue", $@"StgSharp.TerminalDialogue{exeName}");
_dialogueProcessInfo = new ProcessStartInfo
{
FileName = consoleRoute,
Arguments =
$"{_dialogueServerPipe.GetClientHandleAsString()} {_dialogueClientPipe.GetClientHandleAsString()}",
UseShellExecute = true,
RedirectStandardOutput = false,
};
}
internal static void PostMessage(
string instruction,
params string[] operands
)
{
CreateDialogueProcessIfNotExist();
_dw.WriteLine($"{instruction} {operands}");
}
internal static void UnloadDialogue()
{
if (_dialogueProcess != null) {
_dialogueProcess.Kill();
}
_dialogueProcess = null;
_dialogueServerPipe.Dispose();
_dialogueClientPipe.Dispose();
}
internal static string WaitCertainReturn(
string target
)
{
if (_dialogueProcess == null) {
return string.Empty;
}
string temp = string .Empty;
do
{
temp = _dr.ReadLine();
temp = temp ?? string.Empty;
if (!temp!.StartsWith(target))
{
break;
}
} while (true);
int index = temp.IndexOf(' ');
return index != -1 ? temp.Substring(index + 1) : string.Empty;
}
private static void AssertThread()
{
if (_readAndWriteThreadId != Environment.CurrentManagedThreadId) {
throw new InvalidOperationException(
"Communication with dialogue can only call from certain thread.");
}
}
private static void DialogueProcessExitCallback(
object? sender,
EventArgs e
)
{
_dialogueProcess = null;
_dr.Close();
_dw.Close();
}
}
}
@@ -0,0 +1,27 @@
// -----------------------------------------------------------------------------
// file="ErrorDefine"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp
{
public class UniformEmptyReferenceException : Exception
{
public UniformEmptyReferenceException(
string program,
string name
)
: base(
$"Location -1 returned when trying to find uniform {name} in shader program {program}") { }
}
}
@@ -0,0 +1,73 @@
// -----------------------------------------------------------------------------
// file="GlobalSetting"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp
{
public static unsafe partial class World
{
internal const int ssdSegmentLength = 16;
private static byte[] _currentHash = [];
private static bool _inited;
public static byte[] CurrentAssemblyHash
{
get
{
if (_currentHash == null)
{
string route = Assembly.GetExecutingAssembly().Location;
string time = DateTime.UtcNow.ToString();
using (SHA256 sh = SHA256.Create())
{
byte[] bytes = Encoding.UTF8.GetBytes(route);
_currentHash = sh.ComputeHash(bytes);
}
}
return _currentHash;
}
}
public static int MainThreadID
{
get => (_mainThreadID == -1) ?
throw new InvalidOperationException(
"StgSharp environment is not inited. Main thread id is not available") :
_mainThreadID;
internal set => _mainThreadID = value;
}
private static int _mainThreadID { get; set; } = -1;
public static class GlobalSetting
{
private static bool vSyncActivated;
public static bool VSyncActivated
{
get => vsyncActivated;
set =>
// GraphicFramework.glfwSwapInterval(value ? 1 : 0);
vsyncActivated = value;
}
}
}//-------------------------------------- End of Class ---------------------------------------//
}
@@ -0,0 +1,281 @@
// -----------------------------------------------------------------------------
// file="IStaticModule"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
namespace StgSharp
{
/// <summary>
/// Represents a statically activated module that can be registered and initialized as part of
/// the application startup sequence, and gracefully torn down on shutdown.
/// </summary>
public interface IStaticModule
{
/// <summary>
/// Gets the display name of this module, used for logging and diagnostics.
/// </summary>
string ModuleName { get; }
/// <summary>
/// Performs all initialization work for this module using the provided profile.
/// </summary>
/// <param name="profile">
/// The configuration profile supplied at registration time. Use <see
/// cref="IModuleInitializeProfile.Empty" /> if no configuration is required.
/// </param>
void InitializeModule(
IModuleInitializeProfile profile
);
/// <summary>
/// Releases all resources acquired during <see cref="InitializeModule" />. Implementations should
/// be idempotent: calling this method more than once must not throw.
/// </summary>
void UninitializeModule();
}
/// <summary>
/// Marker interface for module initialization configuration. Implementations are plain data
/// carriers; no static factory is required.
/// </summary>
public interface IModuleInitializeProfile
{
/// <summary>
/// A shared, allocation-free empty profile for modules that require no configuration.
/// </summary>
static readonly IModuleInitializeProfile Empty = EmptyModuleInitializeProfile.Instance;
private sealed class EmptyModuleInitializeProfile : IModuleInitializeProfile
{
public static readonly EmptyModuleInitializeProfile Instance = new();
private EmptyModuleInitializeProfile() { }
}
}
/// <summary>
/// Pairs an <see cref="IStaticModule" /> with its <see cref="IModuleInitializeProfile" />,
/// ensuring the two are never stored or iterated independently.
/// </summary>
/// <remarks>
/// Using a <see langword="readonly struct" /> avoids the boxing overhead that would occur if
/// the pair were stored as two parallel lists of reference types.
/// </remarks>
public readonly struct ModuleDescriptor
{
/// <summary>
/// Initializes a new <see cref="ModuleDescriptor" /> using <see
/// cref="IModuleInitializeProfile.Empty" /> as the profile.
/// </summary>
public ModuleDescriptor(
IStaticModule module
)
: this(module, IModuleInitializeProfile.Empty) { }
/// <summary>
/// Initializes a new <see cref="ModuleDescriptor" /> with an explicit profile.
/// </summary>
public ModuleDescriptor(
IStaticModule module,
IModuleInitializeProfile profile
)
{
Module = module;
Profile = profile;
}
/// <summary>
/// Gets the configuration profile associated with the module.
/// </summary>
public IModuleInitializeProfile Profile { get; }
/// <summary>
/// Gets the module to be initialized.
/// </summary>
public IStaticModule Module { get; }
/// <summary>
/// Calls <see cref="IStaticModule.InitializeModule" /> with the stored profile.
/// </summary>
public void Initialize()
{
Module.InitializeModule(Profile);
}
/// <summary>
/// Calls <see cref="IStaticModule.UninitializeModule" /> on the stored module.
/// </summary>
public void Uninitialize()
{
Module.UninitializeModule();
}
}
/// <summary>
/// An ordered collection of modules to be initialized during application startup and
/// uninitialized on shutdown. Supports three registration strategies so that the <c> new()
/// </c> generic constraint is never a hard requirement.
/// </summary>
public sealed class ModuleToInitializeCollection : IEnumerable<IStaticModule>, IStaticModule
{
private readonly List<ModuleDescriptor> _descriptors = [];
internal ModuleToInitializeCollection(
World.Initializer i
)
{
_descriptors.Add(new ModuleDescriptor(i));
}
// ── IEnumerable<IStaticModule> ───────────────────────────────────────────
/// <inheritdoc />
public IEnumerator<IStaticModule> GetEnumerator()
{
foreach (ModuleDescriptor d in _descriptors) {
yield return d.Module;
}
}
// ── Initialization / Uninitialization ────────────────────────────────────
/// <summary>
/// Initializes all registered modules in the order they were registered. Each module
/// receives the profile that was supplied at registration time.
/// </summary>
public void InitializeModule()
{
for (int i = 0; i < _descriptors.Count; i++) {
_descriptors[i].Initialize();
}
}
/// <summary>
/// Uninitializes all registered modules in <b> reverse </b> registration order, mirroring
/// the typical LIFO teardown convention (e.g., GLFW must be unloaded after all subsystems
/// that depend on it).
/// </summary>
public void UninitializeModule()
{
for (int i = _descriptors.Count - 1; i >= 0; i--) {
_descriptors[i].Uninitialize();
}
}
// ── Registration API ─────────────────────────────────────────────────────
/// <summary>
/// Registers a module of type <typeparamref name="T" /> using its default constructor.
/// This is the most concise overload for simple modules that expose a public
/// parameterless constructor.
/// </summary>
/// <typeparam name="T">
/// A concrete module type that implements <see cref="IStaticModule" /> and exposes a
/// public parameterless constructor.
/// </typeparam>
/// <param name="profile">
/// Optional initialization profile. Defaults to <see
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
/// </param>
public ModuleToInitializeCollection UseModule<T>(
IModuleInitializeProfile? profile = null
) where T : IStaticModule, new()
{
return UseModule(static() => new T(), profile);
}
/// <summary>
/// Registers a module produced by a factory delegate. This overload imposes no generic
/// constraints and supports any construction strategy, including dependency injection and
/// parameterized constructors.
/// </summary>
/// <param name="factory">
/// A delegate that constructs and returns the module instance. The delegate is invoked
/// once, immediately at registration time. If <paramref name="factory" /> is <see
/// langword="null" /> or returns <see langword="null" />, the call is silently ignored
/// and the collection is returned unchanged.
/// </param>
/// <param name="profile">
/// Optional initialization profile. Defaults to <see
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
/// </param>
public ModuleToInitializeCollection UseModule(
Func<IStaticModule> factory,
IModuleInitializeProfile? profile = null
)
{
if (factory is null) {
return this;
}
IStaticModule? mod = factory();
if (mod is null) {
return this;
}
_descriptors.Add(new ModuleDescriptor(
mod,
profile ?? IModuleInitializeProfile.Empty));
return this;
}
/// <summary>
/// Registers a pre-existing module instance. Use this overload when the module is
/// provided externally, such as from a DI container, an object pool, or a singleton.
/// </summary>
/// <param name="module">
/// The module instance to register.
/// </param>
/// <param name="profile">
/// Optional initialization profile. Defaults to <see
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
/// </param>
public ModuleToInitializeCollection UseModule(
IStaticModule module,
IModuleInitializeProfile? profile = null
)
{
_descriptors.Add(new ModuleDescriptor(
module,
profile ?? IModuleInitializeProfile.Empty));
return this;
}
/// <inheritdoc />
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
// ── IStaticModule explicit implementation ────────────────────────────────
/// <summary>
/// Explicit <see cref="IStaticModule" /> implementation. Delegates to <see
/// cref="InitializeModule()" />, ignoring the supplied profile because each contained module uses
/// its own registered profile.
/// </summary>
void IStaticModule.InitializeModule(
IModuleInitializeProfile profile
)
{
InitializeModule();
}
/// <inheritdoc />
string IStaticModule.ModuleName => "Module Initializer";
}
}
+220
View File
@@ -0,0 +1,220 @@
// -----------------------------------------------------------------------------
// file="Platform"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp
{
internal enum GraphicAPI : byte
{
GL,
VK
}
public static partial class World
{
/// <summary>
/// Version of current World platform. Main version represent in hundred's digit, and sub
/// version represent in ten's digit.
/// </summary>
public const long version = 430;
private static bool vsyncActivated;
private static uint markCount = 0;
internal static GraphicAPI API = default;
public static ModuleToInitializeCollection Initialize()
{
return new ModuleToInitializeCollection(new Initializer());
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogError(
Exception ex
)
{
DefaultLog.InternalWriteLog(ex.Message, LogType.Error);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogInfo(
string log
)
{
DefaultLog.InternalWriteLog(log, LogType.Info);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogWarning(
string log
)
{
DefaultLog.InternalWriteLog(log, LogType.Warning);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void LogWarning(
Exception notVerySeriesException
)
{
DefaultLog.InternalWriteLog(notVerySeriesException.Message, LogType.Warning);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Mark(
bool needConsole
)
{
markCount++;
if (needConsole) {
Console.WriteLine(markCount);
}
}
public static void Terminate(
int exitCode
)
{
MainTimeProvider.StopProvidingTime();
// GraphicFramework.glfwTerminate();
Environment.Exit(exitCode);
}
public static void Track(
object target,
string message,
bool isConsole,
[CallerFilePath] string filePath = "",
[CallerMemberName] string callerName = "",
[CallerLineNumber] int lineNumber = 0
)
{
Type t = target.GetType();
string log = $"Tracked object: {nameof(target)}\t\tType:{t.Name}\n";
FieldInfo[] fieldInfo = t.GetFields(
BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance);
log += "|\tFieldName\t|\tFieldType\t|\tFieldValue\t|\n";
foreach (FieldInfo field in fieldInfo) {
log += $"|{field.Name}\t\t|{field.FieldType}\t\t|{field.GetValue(target)}\t\t|\n";
}
log += $"Tracking at:\n{filePath}\t{callerName}\tline {lineNumber}\n";
log += $"Additional information:\n{message}\n";
if (isConsole)
{
Console.WriteLine(log);
} else
{
DefaultLog.InternalWriteLog(log, LogType.Info);
}
}
/// <summary>
/// Write a log message to log file, and mark the _level of this message.
/// </summary>
/// <param _label="message">
/// A string contains information of how current evironment runs.
/// </param>
/// <param _label="logType">
/// The severity of current message.
/// </param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void WriteLog(
string message,
LogType logType
)
{
DefaultLog.InternalWriteLog(message, logType);
}
internal static TDele ConvertAPI<TDele>(
IntPtr funcPtr
)
{
if (funcPtr == IntPtr.Zero) {
throw new InvalidCastException(
"Failed to convert api, the function pointer is zero.");
}
try
{
return Marshal.GetDelegateForFunctionPointer<TDele>(funcPtr);
}
catch (Exception)
{
throw;
}
}
internal static Delegate ConvertAPI(
IntPtr funcPtr,
Type T
)
{
if (funcPtr == IntPtr.Zero) {
throw new InvalidCastException(
"Failed to convert api, the function pointer is zero.");
}
try
{
return Marshal.GetDelegateForFunctionPointer(funcPtr, T);
}
catch (Exception)
{
throw;
}
}
internal static unsafe void InternalInitialize()
{
// Initialize time provider and start providing time
MainTimeProvider.StartProvidingTime();
DefaultLog.InternalAppendLog("\n\n\n");
DefaultLog.InternalWriteLog(
$"Program {Assembly.GetEntryAssembly()!.FullName} on {Environment.MachineName} Started.", LogType.Info);
MainThreadID = System.Environment.CurrentManagedThreadId;
/**/
Dialogue.NeedShowWhenStartup = false;
Dialogue.LoadDialogue();
if (Dialogue.NeedShowWhenStartup) {
Dialogue.CreateDialogueProcessIfNotExist();
}
Numa.Initialize();
/**/
}
internal class Initializer : IStaticModule
{
public string ModuleName => "StgSharpCore";
public void InitializeModule(
IModuleInitializeProfile profile
)
{
World.InternalInitialize();
}
public void UninitializeModule()
{
return;
}
}
}
}
+18
View File
@@ -0,0 +1,18 @@
// -----------------------------------------------------------------------------
// file="Global"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
global using StgSharp.Internal;
global using System;
global using System.Runtime.CompilerServices;
global using System.Threading;
[assembly: InternalsVisibleTo("StgSharp.HighPerformance")]
[assembly: InternalsVisibleTo("StgSharp.RegularAnalysis")]
[assembly: InternalsVisibleTo("StgSharp.Graphics")]
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
[assembly: InternalsVisibleTo("StgSharpDebug")]
@@ -0,0 +1,17 @@
// -----------------------------------------------------------------------------
// file="IdMultiplexer"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance
{
public class IdMultiplexer { }
}
@@ -0,0 +1,149 @@
// -----------------------------------------------------------------------------
// file="BufferStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
public unsafe class BufferStack<T> : IDisposable where T : unmanaged
{
private byte* _buffer;
private ulong _count;
public BufferStack(
int capacity
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
AllocateBuffer(capacity);
}
public int Count => (int)_count;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Clear()
{
((Header*)_buffer)->Top = 0;
_count = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Dispose()
{
Marshal.FreeHGlobal((nint)_buffer);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public T Pop()
{
Header* header = (Header*)_buffer;
if (header->Top == 0) {
throw new InvalidOperationException("Stack underflow");
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
return dataStart[header->Top];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Push(
T item
)
{
Header* header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
Expand();
header = (Header*)_buffer;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[header->Top] = item;
header->Top++;
_count++;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryPop(
out T value
)
{
Header* header = (Header*)_buffer;
if (header->Top == 0)
{
value = default;
return false;
}
header->Top--;
_count--;
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[header->Top];
return true;
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(
ReadOnlySpan<T> span
)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart, header->Capacity * (nuint)sizeof(T),
(nuint)span.Length * (nuint)sizeof(T));
}
header->Top = (nuint)span.Length;
_count = (ulong)span.Length;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AllocateBuffer(
int capacity
)
{
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = (nuint)capacity;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void Expand()
{
Header* header = (Header*)_buffer;
nuint newCap = header->Capacity * 2;
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
header = (Header*)_buffer;
header->Capacity = newCap;
}
private struct Header
{
public nuint Capacity;
public nuint Top;
}
}
}
@@ -0,0 +1,299 @@
// -----------------------------------------------------------------------------
// file="ConcurrentBufferStack"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public static class BufferStackBuilder
{
// For BufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static BufferStack<T> Create<T>(
ReadOnlySpan<T> span
) where T : unmanaged
{
if (span.IsEmpty)
{
return new BufferStack<T>(4); // Default small capacity
}
BufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
// For ConcurrentBufferStack collection builder
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static ConcurrentBufferStack<T> CreateConcurrent<T>(
ReadOnlySpan<T> span
) where T : unmanaged
{
if (span.IsEmpty)
{
return new ConcurrentBufferStack<T>(4); // Default small capacity
}
ConcurrentBufferStack<T> stack = new(span.Length);
// Use internal method for efficient bulk initialization
stack.InitializeFromSpan(span);
return stack;
}
}
[CollectionBuilder(
builderType: typeof(BufferStackBuilder),
methodName: nameof(BufferStackBuilder.CreateConcurrent))]
public unsafe class ConcurrentBufferStack<T> : IDisposable where T : unmanaged
{
private byte* _buffer;
private readonly BufferExpansionLock _lock = new();
private long _count;
public ConcurrentBufferStack()
{
AllocateBuffer(4);
}
public ConcurrentBufferStack(
int capacity
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
AllocateBuffer(capacity);
}
public ulong Count => (ulong)Volatile.Read(ref _count);
public void Clear()
{
_lock.EnterExpansionProcess();
try
{
_lock.EnterBufferCopy(); // exclusive while mutating Top
try
{
Header* header = (Header*)_buffer;
header->Top = 0;
Volatile.Write(ref _count, 0);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
public void Dispose()
{
_lock.Dispose();
NativeMemory.Free(_buffer);
}
public T Pop()
{
while (true)
{
_lock.EnterBufferRead(); // prevent buffer being reallocated while reading/modifying Top
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0) {
throw new InvalidOperationException("Stack underflow");
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
_ = Interlocked.Decrement(ref _count);
return dataStart[oldTop - 1];
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
public void Push(
T item
)
{
while (true)
{
// Ensure capacity, expanding if needed
_lock.EnterMetaDataRead();
try
{
Header* header = (Header*)_buffer;
int topSnapshot = Volatile.Read(ref header->Top);
int cap = Volatile.Read(ref header->Capacity);
if (topSnapshot >= cap)
{
// Need expansion
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
header = (Header*)_buffer;
if (header->Top >= header->Capacity)
{
int newCap = header->Capacity * 2;
nuint newSize = ((nuint)sizeof(Header)) + (((nuint)newCap) * ((nuint)sizeof(T)));
_lock.EnterBufferCopy();
try
{
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
((Header*)_buffer)->Capacity = newCap;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
finally
{
_lock.ExitExpansionProcess();
}
continue;
}
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
// Reserve and write under exclusive buffer lock to avoid consumers reading unwritten data
_lock.EnterBufferCopy();
try
{
Header* header = (Header*)_buffer;
int idx = header->Top; // reserve next slot
if (idx >= header->Capacity)
{
// lost race against expander or other producer; retry
continue;
}
T* dataStart = (T*)(_buffer + sizeof(Header));
dataStart[idx] = item; // write first
header->Top = idx + 1; // then publish by increasing top
_ = Interlocked.Increment(ref _count);
return;
}
finally
{
_lock.ExitBufferCopy();
}
}
}
public bool TryPop(
out T value
)
{
while (true)
{
_lock.EnterBufferRead();
try
{
Header* header = (Header*)_buffer;
int oldTop = Volatile.Read(ref header->Top);
if (oldTop <= 0)
{
value = default;
return false;
}
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
{
T* dataStart = (T*)(_buffer + sizeof(Header));
value = dataStart[oldTop - 1];
_ = Interlocked.Decrement(ref _count);
return true;
}
}
finally
{
_lock.ExitBufferRead();
}
Thread.Sleep(0);
}
}
// Internal method for efficient bulk initialization
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal unsafe void InitializeFromSpan(
ReadOnlySpan<T> span
)
{
if (span.IsEmpty) {
return;
}
Header* header = (Header*)_buffer;
T* dataStart = (T*)(_buffer + sizeof(Header));
fixed (T* sourcePtr = span) {
Buffer.MemoryCopy(sourcePtr, dataStart,
((nuint)header->Capacity) * ((nuint)sizeof(T)),
((nuint)span.Length) * ((nuint)sizeof(T)));
}
header->Top = span.Length;
Volatile.Write(ref _count, span.Length);
}
private void AllocateBuffer(
int capacity
)
{
nuint size = ((nuint)sizeof(Header)) + (((nuint)capacity) * ((nuint)sizeof(T)));
_buffer = (byte*)NativeMemory.Alloc(size);
Header* header = (Header*)_buffer;
header->Top = 0;
header->Capacity = capacity;
}
private struct Header
{
public int Capacity; // max capacity of the stack (number of elements)
public int Top; // next available index in the stack
}
}
}
@@ -0,0 +1,159 @@
# TLSF (Two-Layer Segregated Fit) Allocator
This document describes the current StgSharp native arena allocator, `StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator`. The file name is historical; the implementation described below is TLSF rather than the older HLSF allocator.
## Overview
- Single contiguous native arena allocated up front
- Headerless returned blocks; allocator metadata is stored in 32-byte nodes carved from the tail of the same arena
- Requests rounded up to 64-byte granularity, with 64 bytes as the minimum block size
- Free blocks grouped by coarse size level derived from the highest set bit of the block size
- Exact-level allocation uses first-fit traversal inside the selected bucket; higher levels fall back to the head of the first non-empty bucket
- `Free` performs bounded local coalescing; `Collect` performs full free-run coalescing across the arena
- Single-threaded implementation; concurrent access requires external synchronization
- No arena growth and no implicit fallback to `NativeMemory.AlignedAlloc` for oversized requests
## Quick Use
```csharp
using StgSharp.HighPerformance.Memory;
using var tlsf = new TwoLayerSegregatedFitAllocator(64 * 1024 * 1024);
var handle = tlsf.Alloc(1024);
Span<byte> buffer = handle.AsSpan();
buffer[0] = 42;
tlsf.Free(handle);
```
## Constructors
- `TwoLayerSegregatedFitAllocator(nuint byteSize)`
- `TwoLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)`
- `TwoLayerSegregatedFitAllocator(nuint byteSize, int align)`
- `TwoLayerSegregatedFitAllocator(nuint byteSize, int align, nuint maxBlockSize)`
Behavior summary:
- `byteSize` defines the total arena size
- `maxBlockSize` defaults to 32 MB and defines the largest single request accepted by `Alloc` and `TryAlloc`
- `align` is passed to the backing `NativeMemory.AlignedAlloc` call for arena allocation; request sizes inside the arena are still quantized to 64-byte blocks
## Public API
### `Alloc(nuint size)`
- Rounds `size` up to the next 64-byte block, with a minimum of 64 bytes
- Throws `OverflowException` if the normalized size exceeds `maxBlockSize` or if the arena cannot provide a block
- Returns a `Handle` containing block address and block size
- Splits a sufficiently large tail remainder into a new free node and re-buckets that node
### `TryAlloc(nuint size, out Handle handle)`
- Uses the same size normalization and maximum block limit as `Alloc`
- Returns `false` when the request exceeds the configured limit or when the arena is exhausted
- Does not fall back to the system allocator
### `Free(Handle handle, int collectScanWindow = 4)`
- Marks the block as free
- Scans up to `collectScanWindow` neighbors toward lower addresses and then up to `collectScanWindow` neighbors toward higher addresses
- Merges any contiguous free neighbors found during those scans
- Reclassifies the merged block back into the coarse bucket set
### `Collect()`
- Walks the full position chain from low address to high address
- Merges consecutive free runs without the local scan-window limit used by `Free`
- Restores larger reusable spans and improves bucket coverage after fragmentation-heavy phases
### `Dispose()`
- Releases the backing arena through the stored free callback
### `Handle`
- Readonly value type carrying the allocator metadata reference, block address, and block size
- `Address` exposes the native pointer
- `Size` exposes the normalized block size
- `AsSpan(int range = -1)` exposes a span view over the block
- The indexer provides byte-level access by offset
## Allocation Model
### Level Classification
- `GetLevel(size)` maps the normalized size to `57 - BitOperations.LeadingZeroCount(size)` and clamps the result to `_maxLevel`
- Each bucket corresponds to one coarse size class rather than an exact size
- The target bucket can therefore contain blocks of different sizes within the same coarse level
### Allocation Path
1. Normalize the requested size to a 64-byte multiple.
2. Traverse the target bucket and select the first free block whose size is at least the requested size.
3. If the target bucket has no suitable block, pop the head of the first non-empty higher-level bucket.
4. If all buckets are empty, carve a fresh block from the spare region tracked by the arena sentinel.
5. Split the tail remainder when both remainder size and metadata availability allow it.
### Metadata Layout
- `Metadata` is a 32-byte node
- `NextNear` and `PreviousNear` form the address-ordered position chain
- `NextLevel` and `PreviousLevel` form either the bucket chain or the recycled-metadata chain, depending on node state
- The spare sentinel at the end of the arena tracks remaining virgin space, recycled metadata nodes, and position-chain anchors
- Returned blocks remain headerless from the caller's point of view
### Coalescing Model
- `Free` performs bounded local coalescing to keep the hot path short
- `Collect` performs full coalescing of consecutive free runs across the entire position chain
- Recycled metadata nodes are returned to the spare metadata list and reused by later splits or fresh arena chunks
## Complexity and Runtime Characteristics
- `Alloc`: O(1) bucket index calculation plus target-level first-fit traversal; higher-level fallback is O(1)
- `Free`: O(`collectScanWindow`) local merge cost
- `Collect`: O(number of metadata nodes in the position chain)
- Best steady-state behavior appears on repeated small and medium native allocations inside a pre-allocated arena
## BenchmarkDotNet (TLSF vs Libc)
Environment: .NET 8.0.26 (PerformanceDebug), x64 RyuJIT AVX2, Windows 11, AMD Ryzen 7 6800H, BenchmarkDotNet v0.15.8, InProcessEmitToolchain, LaunchCount=1, WarmupCount=6, IterationCount=15, InvocationCount=3.
Workload summary:
- Arena size: 512 MB
- Arena alignment: 64 bytes
- Allocation count per batch: 2048
- Batch count: 256
- Request set before normalization: 64, 100, 200, 400, 1024, 4096 bytes
- Free order: shuffled per batch
- `Free` local merge window: 4
Best observed run under the current harness:
| Method | Mean | Error | StdDev | Ratio | RatioSD | Allocated | Alloc Ratio |
|------- |-----:|------:|-------:|------:|--------:|----------:|------------:|
| Tlsf | 79.11 ns | 5.944 ns | 5.560 ns | 0.65 | 0.09 | - | NA |
| Libc | 123.64 ns | 15.955 ns | 14.144 ns | 1.01 | 0.16 | - | NA |
Worst observed run under the same harness:
| Method | Mean | Error | StdDev | Ratio | RatioSD | Allocated | Alloc Ratio |
|------- |-----:|------:|-------:|------:|--------:|----------:|------------:|
| Tlsf | 82.95 ns | 6.337 ns | 5.617 ns | 0.81 | 0.07 | - | NA |
| Libc | 102.63 ns | 8.590 ns | 7.173 ns | 1.00 | 0.09 | - | NA |
Interpretation:
- The current single-pass bucket selection fast path materially reduces allocator-side latency
- The reported ratio still drifts across repeated compile-and-run cycles because the libc baseline also moves under the current InProcess harness
- Recent repeated runs place the transient best case near 0.65 and the longer-running steady band near 0.75 to 0.80
## Operational Notes
- Maximum single-allocation size is fixed at construction time
- Arena growth is not implemented
- `Collect` is the global defragmentation pass; `Free` only performs bounded local coalescing
- The allocator is intended for explicit native-memory workflows rather than as a drop-in general-purpose heap replacement
@@ -0,0 +1,97 @@
// -----------------------------------------------------------------------------
// file="L4.CacheLine"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.ProcessorAbstraction;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
{
internal enum AllocationState : byte
{
SpareOrCollect = 0,
Free = 1,
Alloc = 2,
}
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
private unsafe struct CacheLineMetadata
{
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
[FieldOffset(0)] private readonly M256 FillingMask;
[FieldOffset(24)] internal nuint Origin;
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
#region tlsf linked list
[FieldOffset(0)] public short NextLevel;
[FieldOffset(2)] public short NextNear;
[FieldOffset(4)] public short PreviousLevel;
[FieldOffset(6)] public short PreviousNear;
#endregion
#region buffer
[FieldOffset(8)] public uint PositionMask;
[FieldOffset(12)] public uint SizeMask;
#endregion
#region mask
[FieldOffset(16)] internal AllocationState State;
[FieldOffset(17)] internal byte ReversedProfile;
[FieldOffset(18)] internal ushort CustomizeProfile;
[FieldOffset(20)] internal int RefCount;
#endregion
}
#pragma warning disable IDE0032
[StructLayout(LayoutKind.Explicit)]
public unsafe struct Handle : IDisposable
{
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
[FieldOffset(0)] private nuint _handle;
private Handle(
nuint handle,
CacheLineMetadata* cacheRefCount
)
{
_handle = handle;
_head = cacheRefCount;
}
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
public Handle() { }
public readonly nuint Address => _handle;
public void Dispose()
{
if (_handle == 0) {
return;
}
_handle = 0;
_ = Interlocked.Decrement(ref _head->RefCount);
}
}
#pragma warning restore IDE0032
}
}
@@ -0,0 +1,92 @@
// -----------------------------------------------------------------------------
// file="L4.Debug"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
/**/
public partial class L4
{
public void DebugAgeAll()
{
// AgeAll();
}
public bool DebugContains(
nuint origin
)
{
return _map.TryGet(origin, out _);
}
public int DebugEvict(
int count = 1
)
{
return Evict(count);
}
public unsafe DebugSnapshot GetDebugSnapshot()
{
throw new NotImplementedException();
/*
int maxId = _maxId;
byte[] rrpv = new byte[maxId];
byte* p = (byte*)_eviction;
for (int i = 0; i < maxId; i++) {
rrpv[i] = p[i];
}
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
/**/
}
public readonly struct DebugSnapshot
{
/// <summary>
/// RRPV value for each line(0~3), length == MaxId
/// </summary>
public readonly byte[] RrpvSnapshot;
/// <summary>
/// Count of activated cache line handle
/// </summary>
public readonly int ActiveCount;
/// <summary>
/// Count of predictor ahead of current map position(aheadCount)
/// </summary>
public readonly int AheadCount;
/// <summary>
/// Max ID of the line being using or once used(maxId)
/// </summary>
public readonly int MaxId;
internal DebugSnapshot(
int activeCount,
int aheadCount,
int maxId,
byte[] rrpv
)
{
ActiveCount = activeCount;
AheadCount = aheadCount;
MaxId = maxId;
RrpvSnapshot = rrpv;
}
}
}
/**/
}
@@ -0,0 +1,64 @@
// -----------------------------------------------------------------------------
// file="L4.Evict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
{
/// <summary>
///
/// </summary>
/// <param name="count">
///
/// </param>
/// <returns>
/// Count of lines not evicted
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe int Evict(
int count = 1
)
{
count = (count < 0) ? 0 : count;
int* pPtr = _predictCount, mPtr = _mapCount;
int remain = count;
for (int i = 0; i < CacheLineCount; i++)
{
if (pPtr[i] > mPtr[i])
{
continue;
}
remain -= TryEvictAt(i) ? 1 : 0;
}
return remain;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe bool TryEvictAt(
int index
)
{
CacheLineMetadata* head = &_head[index];
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
{
// EvictLine(i + index);
_ = _map.TryRemove(head->Origin, out _);
IL4Predict predict = CurrentPredict;
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
_bufferAllocator.Free(index);
_mapCount[index] = 0;
_predictCount[index] = int.MaxValue;
return true;
}
return false;
}
}
}
@@ -0,0 +1,34 @@
// -----------------------------------------------------------------------------
// file="L4.Exceptions"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
namespace StgSharp.HighPerformance.Memory
{
#pragma warning disable CA1032 // Implement standard exception constructors
public sealed class L4PredictUnregisteredException : Exception
{
public L4PredictUnregisteredException(
int id
)
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
}
public sealed class L4PredictConflictException : Exception
{
public L4PredictConflictException(
nuint basePtr,
ushort policy,
int current
)
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
}
#pragma warning restore CA1032 // Implement standard exception constructors
}
@@ -0,0 +1,38 @@
// -----------------------------------------------------------------------------
// file="L4.Predict"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
{
private IL4Predict CurrentPredict { get; set; }
public void ResetPredict(
IL4Predict predict
)
{
if (CurrentPredict is null)
{
CurrentPredict = predict;
return;
}
// TODO: write back all and register new predictor
}
}
}
@@ -0,0 +1,146 @@
// -----------------------------------------------------------------------------
// file="L4.Predictor"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
[Flags]
public enum PredictResult
{
Success = 1,
Exhausted = 2,
}
public interface IL4Predict
{
PredictResult Predict(
Span<CacheLineDescription> node,
out int count
);
/// <summary>
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
/// name="cache" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Destination cache line buffer.
/// </param>
void Prefetch(
nuint origin,
uint policy,
Span<byte> cache
);
/// <summary>
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
/// </summary>
/// <param name="origin">
/// Source address associated with the cache line.
/// </param>
/// <param name="policy">
/// Mapping policy associated with the cache line.
/// </param>
/// <param name="cache">
/// Source cache line buffer.
/// </param>
void WriteBack(
nuint origin,
uint policy,
ReadOnlySpan<byte> cache
);
}
[StructLayout(LayoutKind.Explicit)]
public struct CacheLineDescription : IEquatable<CacheLineDescription>
{
public override readonly bool Equals(
object? obj
)
{
return (obj is CacheLineDescription prediction) &&
Address.Equals(prediction.Address) &&
(MapPolicy == prediction.MapPolicy);
}
public readonly bool Equals(
CacheLineDescription other
)
{
return
(Address == other.Address) && (MapPolicy == other.MapPolicy);
}
public override readonly int GetHashCode()
{
return (int)ComposePredictionKey64(Address, MapPolicy);
}
public readonly nuint GetHashCodeLong()
{
return ComposePredictionKey64(Address, MapPolicy);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static nuint ComposePredictionKey64(
nuint address,
ushort mapPolicy
)
{
nuint x = address;
nuint y = mapPolicy;
unchecked
{
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
mixed ^= mixed >> 30;
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
mixed ^= mixed >> 27;
mixed *= (nuint)0x94d049bb133111ebUL;
mixed ^= mixed >> 31;
return mixed;
}
}
public static bool operator !=(
CacheLineDescription left,
CacheLineDescription right
)
{
return !(left == right);
}
public static bool operator ==(
CacheLineDescription left,
CacheLineDescription right
)
{
return left.Equals(right);
}
#pragma warning disable CA1051
[FieldOffset(0)]public nuint Address;
[FieldOffset(8)]public uint Size;
[FieldOffset(12)]public ushort MapPolicy;
[FieldOffset(14)]internal short Predictor;
#pragma warning restore CA1051
}
}
@@ -0,0 +1,102 @@
// -----------------------------------------------------------------------------
// file="L4.Prefetch"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.ComponentModel;
using System.Numerics;
namespace StgSharp.HighPerformance.Memory
{
public unsafe partial class L4
{
private partial void Prefetch()
{
if (CurrentPredict is null) {
return;
}
IL4Predict predict = CurrentPredict;
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
PredictResult result = predict.Predict(span, out int actualCount);
_exhausted = (result & PredictResult.Exhausted) != 0;
if (actualCount <= 0)
{
_exhausted = true;
return;
}
int missIndex = 0;
for (int i = 0; i < actualCount; i++)
{
ref CacheLineDescription handle = ref span[i];
nuint address = handle.Address;
if (!_map.TryGet(address, out _))
{
// miss and record
span[missIndex] = handle;
missIndex++;
}
}
// now prefetch the misses
for (int i = 0; i < missIndex; i++)
{
CacheLineMetadata* head;
ref CacheLineDescription pHandle = ref span[i];
nuint address = pHandle.Address;
int index;
if (_map.TryGet(address, out nuint handle))
{
head = (CacheLineMetadata*)handle;
index = (int)(head - _head);
_predictCount[index] = 0; // reset eviction counter
#region update predictor belonging
if (_predictCount[index] > short.MaxValue / 2)
{
int predictCount = Volatile.Read(ref _predictCount[index]);
int mapCount = Volatile.Read(ref _mapCount[index]);
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
_ = Interlocked.Add(ref _predictCount[index], -decrement);
_ = Interlocked.Add(ref _mapCount[index], -decrement);
}
_predictCount[index]++;
#endregion
continue;
}
int retryCount = 0, remain = missIndex - i;
const int maxRetry = 1024;
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
{
remain = Evict(remain);
retryCount++;
if (retryCount > maxRetry) {
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
}
if (retryCount > maxRetry / 2) {
Thread.Sleep(1);
}
}
head = _head + index;
head->Origin = address;
head->CustomizeProfile = pHandle.MapPolicy;
head->RefCount = 0;
_predictCount[index] = 1;
_mapCount[index] = 1;
// pre fetch them here
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
_ = _map.TryAddOrSet(address, (nuint)head, out _);
}
_ = Interlocked.Add(ref _aheadCount, actualCount);
}
}
}
@@ -0,0 +1,519 @@
// -----------------------------------------------------------------------------
// file="L4.Tlsf"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
public partial class L4
{
private unsafe class TLSF
{
public const int MaxCacheLineCount = 8192;
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
private readonly short[] _bucket;
private readonly CacheLineMetadata* _spareMemory;
private readonly int _align;
private readonly int _maxLevel;
private int _metadataEnd;
private readonly Ptr64 _basePtr;
private readonly uint _maxAllocSize;
private readonly uint _totalSize;
internal TLSF(
uint byteSize,
uint maxBlockSize,
void* externalPointer
)
{
_totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer;
_align = 4096; // align of a memory page
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new short[GetLevel(_maxAllocSize) + 2];
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
_metadataEnd = 0;
// init spare memory node
_spareMemory->PositionMask = 0;
_spareMemory->SizeMask = _totalSize;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0;
_spareMemory->PreviousNear = 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(
uint size
)
{
int origin = 25 - BitOperations.LeadingZeroCount(size);
return int.Min(origin, _maxLevel);
}
/// <summary>
/// Put a metadata node back to free list and waiting for next allocation.
/// </summary>
/// <param name="node">
///
/// </param>
private void RecycleNode(
CacheLineMetadata* node
)
{
short nodeIdx = (short)(_spareMemory - node);
int level = GetLevel(node->SizeMask);
short nextLevelIdx = node->NextLevel;
short prevLevelIdx = node->PreviousLevel;
RemoveFromBucket(level, node, nodeIdx);
// remove from position chain
short prevNearIdx = node->PreviousNear;
short nextNearIdx = node->NextNear;
CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
prevNear->NextNear = nextNearIdx;
if (nextNearIdx != 0)
{
CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
nextNear->PreviousNear = prevNearIdx;
} else
{
_spareMemory->PreviousNear = prevNearIdx;
}
node->PreviousNear = 0;
node->NextNear = 0;
node->PositionMask = 0;
node->SizeMask = 0;
node->State = AllocationState.SpareOrCollect;
node->PreviousLevel = 0;
short nextIdx = _spareMemory->NextLevel;
if (nextIdx != 0)
{
CacheLineMetadata* next = &_spareMemory[nextIdx];
next->PreviousLevel = nodeIdx;
}
node->NextLevel = nextIdx;
_spareMemory->NextLevel = nodeIdx;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(
CacheLineMetadata* node,
int nodeIdx
)
{
RemoveFromBucket(GetLevel(node->SizeMask), node, nodeIdx);
}
private void RemoveFromBucket(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{
int currentLevel = level;
short currentNextLevelIdx = node->NextLevel;
short currentPrevLevelIdx = node->PreviousLevel;
if (_bucket[currentLevel] == nodeIdx)
{
_bucket[currentLevel] = currentNextLevelIdx;
} else if (currentPrevLevelIdx != 0)
{
CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
prevLevel->NextLevel = currentNextLevelIdx;
}
if (currentNextLevelIdx != 0)
{
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
nextLevel->PreviousLevel = currentPrevLevelIdx;
}
node->PreviousLevel = 0;
node->NextLevel = 0;
}
// Assuming node to remove is head f bucket
private void RemoveFromBucketHead(
int level,
CacheLineMetadata* node,
int nodeIdx
)
{
int currentLevel = level;
short currentNextLevelIdx = node->NextLevel;
// int currentPrevLevelIdx = 0;
if (_bucket[currentLevel] == nodeIdx)
{
_bucket[currentLevel] = currentNextLevelIdx;
}
/*
else if (currentPrevLevelIdx != 0)
{
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
prevLevel->NextLevel = currentNextLevelIdx;
}
/**/
if (currentNextLevelIdx != 0)
{
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
nextLevel->PreviousLevel = 0;
}
node->PreviousLevel = 0;
node->NextLevel = 0;
}
private bool TryAllocNewChunk(
uint size,
out CacheLineMetadata* meta
)
{
short idx = _spareMemory->NextLevel;
uint spareSize = _spareMemory->SizeMask;
if (idx != 0)
{
if (spareSize < size)
{
meta = null;
return false;
}
meta = &_spareMemory[idx];
short nextSpareIdx = meta->NextLevel;
if (nextSpareIdx != 0)
{
CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
nextSpare->PreviousLevel = 0;
}
_spareMemory->NextLevel = nextSpareIdx;
} else
if (!TryGetNewMetadata(out meta)) {
return false;
}
spareSize -= size;
meta->SizeMask = size;
meta->PositionMask = _spareMemory->PositionMask;
meta->State = AllocationState.Free;
meta->PreviousLevel = 0;
meta->NextLevel = 0;
_spareMemory->SizeMask = spareSize;
_spareMemory->PositionMask += size;
// insert the new chunk to position chain
short nearIdx = _spareMemory->PreviousNear;
CacheLineMetadata* near = &_spareMemory[nearIdx];
near->NextNear = idx;
meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = idx;
meta->NextNear = 0;
return true;
}
private bool TryGetNewMetadata(
out CacheLineMetadata* meta
)
{
ulong spareSize = _spareMemory->SizeMask;
if (_metadataEnd >= MaxCacheLineCount)
{
meta = null;
return false;
}
_metadataEnd++;
meta = &_spareMemory[_metadataEnd];
return true;
}
private bool TryTakeBucketEntry(
uint minimumSize,
out CacheLineMetadata* entry
)
{
int level = GetLevel(minimumSize);
// first try to find in the target level
{
int idx = _bucket[level];
while (idx != 0)
{
CacheLineMetadata* candidate = &_spareMemory[idx];
if (candidate->SizeMask >= minimumSize)
{
RemoveFromBucket(level, candidate, idx);
entry = candidate;
return true;
}
idx = candidate->NextLevel;
}
}
for (int i = level + 1; i < _bucket.Length; i++)
{
int idx = _bucket[i];
if (idx != 0)
{
CacheLineMetadata* candidate = &_spareMemory[idx];
RemoveFromBucketHead(i, candidate, idx);
entry = candidate;
return true;
}
}
entry = null;
return false;
}
#region alloc and free
public int Alloc(
uint size
)
{
uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
}
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
!TryAllocNewChunk(allocSize, out entry)) {
throw new OverflowException("No available ");
}
short entryIdx = (short)(_spareMemory - entry);
uint entrySize = entry->SizeMask;
if (entrySize < allocSize) {
throw new OverflowException("Bucket returned an undersized block.");
}
uint remain = entrySize - allocSize;
// slice tail
if (remain > ulong.Min(4096UL, entrySize >> 7) &&
TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = newEntryIdx;
}
newEntry->NextLevel = bucketHeadIdx;
newEntry->PreviousLevel = 0;
_bucket[newLevel] = newEntryIdx;
// push to position chain
short afterIdx = entry->NextNear;
CacheLineMetadata* after = &_spareMemory[afterIdx];
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
return entryIdx;
}
public void Collect()
{
short currentIdx = _spareMemory->NextNear;
do
{
if (currentIdx == 0)
{
break;
}
CacheLineMetadata* current = &_spareMemory[currentIdx];
if (current->State == AllocationState.Free)
{
// scan next
uint size = 0;
int nextIdx = current->NextNear;
while (nextIdx != 0)
{
CacheLineMetadata* next = &_spareMemory[nextIdx];
if (next->State != AllocationState.Free)
{
break;
}
size += next->SizeMask;
RecycleNode(next);
nextIdx = current->NextNear;
}
if (size > 0)
{
current->SizeMask += size;
RemoveFromBucket(current, currentIdx);
int newLevel = GetLevel(current->SizeMask);
short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = currentIdx;
}
current->NextLevel = bucketHeadIdx;
current->PreviousLevel = 0;
_bucket[newLevel] = currentIdx;
}
}
currentIdx = current->NextNear;
} while (true);
}
public void Free(
int handle,
int collectScanWindow = 4
)
{
CacheLineMetadata* head = &_spareMemory[handle];
short headIdx = (short)(_spareMemory - head);
uint size = head->SizeMask;
for (int i = 0; i < collectScanWindow; i++)
{
// forward scan, previous direction
int forwardIdx = head->PreviousNear;
if (forwardIdx == 0)
{
break;
}
CacheLineMetadata* prev = &_spareMemory[forwardIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->SizeMask;
head->PositionMask = prev->PositionMask;
RecycleNode(prev);
}
for (int i = 0; i < collectScanWindow; i++)
{
// backward scan, next direction
int nextIdx = head->NextNear;
if (nextIdx == 0)
{
break;
}
CacheLineMetadata* prev = &_spareMemory[nextIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->SizeMask;
RecycleNode(prev);
}
// recycle to level;
head->SizeMask = size;
head->State = AllocationState.Free;
head->PreviousLevel = 0;
int level = GetLevel(head->SizeMask);
short bucketHeadIdx = _bucket[level];
head->NextLevel = 0;
if (bucketHeadIdx != 0)
{
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
bucketHead->PreviousLevel = headIdx;
head->NextLevel = bucketHeadIdx;
}
_bucket[level] = headIdx;
return;
}
public bool TryAlloc(
uint size,
out int h
)
{
uint allocSize = uint.Max(64, (~63U) & (size + 63));
if (allocSize > _maxAllocSize)
{
h = -1;
return false;
}
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
!TryAllocNewChunk(allocSize, out entry))
{
h = -1;
return false;
}
short entryIdx = (short)(_spareMemory - entry);
uint entrySize = entry->SizeMask;
if (entrySize < allocSize)
{
h = -1;
return false;
}
uint remain = entrySize - allocSize;
// slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
{
short newEntryIdx = (short)(_spareMemory - newEntry);
newEntry->PositionMask = entry->PositionMask + allocSize;
newEntry->SizeMask = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
short bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
newEntry->NextLevel = bucketHeadIdx;
bucketHead->PreviousLevel = newEntryIdx;
} else
{
newEntry->NextLevel = 0;
}
newEntry->PreviousLevel = 0;
_bucket[newLevel] = newEntryIdx;
// push to position chain
short afterIdx = entry->NextNear;
CacheLineMetadata* after = &_spareMemory[afterIdx];
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->PositionMask - (ulong)_basePtr > _totalSize)
{
h = default;
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
h = entryIdx;
return true;
}
#endregion
}
}
}
@@ -0,0 +1,188 @@
// -----------------------------------------------------------------------------
// file="L4"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.HighPerformance.Memory;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System.Data;
using System.Runtime.InteropServices;
namespace StgSharp.HighPerformance.Memory
{
public sealed unsafe partial class L4 : IDisposable
{
private const int CacheLineCount = 8192;
private const nuint PageSize = 4096;
private const int PredictionCount = 2048;
private const int PredictionLeastAhead = 1024;
/// <summary>
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
/// </summary>
private readonly CacheLineMetadata* _head;
/// <summary>
/// Count of cache lines mapped for each prediction, used for eviction policy.
/// </summary>
private readonly int* _mapCount;
/// <summary>
/// Count of times each cache line is predicted, used for eviction policy.
/// </summary>
private readonly int* _predictCount;
/// <summary>
/// Original memory view for prediction span.
/// </summary>
private readonly CacheLineDescription* _prediction;
private bool _disposed;
private bool _exhausted;
private int _aheadCount;
private readonly int _bindedNuma;
private volatile int _prefetching;
private readonly nuint _baseAddress;
private readonly SwissTable _map;
private readonly TLSF _bufferAllocator;
public L4(
int bindedNuma,
uint bufferSize = 8192 * 8192,
int align = 16,
uint maxCacheLineSize = 8192
)
{
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
_bindedNuma = bindedNuma;
// init order is fallow by align requirement of each component.
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
ByteCapacity = normalizedBufferSize;
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
// malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance.
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
nuint ptr = _baseAddress;
// init all components with the buffer, and move the buffer pointer forward.
nuint _buffer = _baseAddress;
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
_buffer += normalizedBufferSize;
_head = (CacheLineMetadata*)ptr;
ptr += headSize;
_prediction = (CacheLineDescription*)ptr;
ptr += predictionSize;
_map = SwissTable.Create(ptr, _ => { });
ptr += mapCountSize;
_predictCount = (int*)ptr;
ptr += predictionCountSize;
_mapCount = (int*)ptr;
ptr += predictionCountSize;
M512* mPtr = (M512*)_head;
for (uint i = 1; i < CacheLineCount; i++) {
mPtr->BroadCastFrom<ulong>(0);
}
}
public nuint ByteCapacity { get; init; }
public nuint AllocSize { get; init; }
// public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
Handle line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design",
"CA1063:Implement IDisposable correctly",
Justification = """
Only owns native memory via Numa.Free();
no managed IDisposable fields.
Dispose(bool) pattern is unnecessary.
""")]
public void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_map.Dispose();
#pragma warning disable CA1816
GC.SuppressFinalize(_bufferAllocator);
#pragma warning restore CA1816
// init order is fallow by align requirement of each component.
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public Handle Map(
ref CacheLineDescription prediction
)
{
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
{
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{
try
{
if (_aheadCount < PredictionLeastAhead)
{
// do prefetch in work load now, sync
Prefetch();
}
}
finally
{
_prefetching = 0;
}
}
}
while (true)
{
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
{
// cache hit
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
if (Interlocked.Increment(ref entry->RefCount) > 0)
{
_predictCount[entry - _head] = 0;
_ = Interlocked.Decrement(ref _aheadCount);
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
return *(Handle*)hPtr;
} else
{
_ = Interlocked.Decrement(ref entry->RefCount);
}
}
Thread.Sleep(0);
}
}
private partial void Prefetch();
}
}
@@ -0,0 +1,762 @@
# L4 Software Cache Design Specification
> Project: StgSharp
> Author: Nitload
> Version: Draft v0.3
> Date: 2026-03-10
---
## 1. Purpose
### 1.1 Problem Background
StgSharp's matrix computation system is migrating from a "single global context table" to a
"per-thread context" architecture to adapt to the scheduling characteristics of hybrid-architecture
CPUs (e.g., Intel P-Core / E-Core). After the migration, each thread independently holds its own
computation context, but matrix data may still be allocated by the HLSF allocator or global
`malloc` on a remote NUMA node, resulting in significantly increased cross-node access latency.
At the same time, in matrix multiplication (FMA), the core row data of matrix A in the
checkerboard-shaped kernel grid layout must be repacked (Packed) into a micro-kernel-friendly
contiguous arrangement. In libraries such as OpenBLAS, this Pack operation is re-executed on
every tile switch, whereas L4 can cache the Pack results to avoid repeated reordering.
Additionally, the segmented accumulation (segment) of matrix multiplication requires storing
intermediate partial sums locally to avoid cross-NUMA writes to the target matrix on every
segment. L4 can serve as a local accumulation buffer for partial sums, writing them back in a
single operation after all segments complete locally.
### 1.2 Solution
The L4 cache is a purely software-implemented **data caching layer** logically positioned
between the CPU hardware caches (L1/L2/L3) and main memory. Its core responsibilities:
1. **Pack result caching**: Cache the repack results of FMA core rows in matrix A to avoid repeated Pack operations
2. **NUMA migration cache**: Migrate data from remote NUMA nodes to local memory and cache it, eliminating cross-node latency
3. **Segment accumulation buffer**: Temporarily store partial sums from FMA segmented computation to reduce remote write operations
4. **PLU intermediate data buffer**: Provide a local contiguous workspace for PLU decomposition
### 1.3 Design Goals
- Provide NUMA-friendly local data access for per-thread matrix computation contexts
- Fixed capacity (64 MB), no runtime expansion, deterministic latency
- Transparent pointer redirection: upper-layer code queries via original pointers; L4 returns local cache pointers
- Cache eviction policy based on **RRPV mask + windowed OPT prediction**
- Support both **StreamTransparent** and **CacheBuffer** working modes
- Support **writable cache lines** that invoke registered callback functions for pre-processing and write-back upon eviction or explicit flush
---
## 2. Overall Architecture
### 2.1 Three-Component Model
```
┌���─────────────────────────────────────────────────────────────────┐
│ L4 │
│ │
│ ┌──────────────┐ ┌────────────────────┐ ┌──────────────────┐ │
│ │ CacheLine │ │ L4 Core │ │ Predictor │ │
│ │ Handle │ │ Cache │ │ Prefetch │ │
│ │ │ │ │ │ Predictor │ │
│ │ • local addr │ │ • Swiss Table │ │ • 2048-entry buf │ │
│ │ • ref count │ │ • Slab Allocator │ │ • threshold 1024 │ │
│ │ • Dispose │ │ • RRPV mask │ │ • windowed OPT │ │
│ │ drops ref │ │ • SIMD eviction │ │ • active key set │ │
│ │ │ │ • writeback mgmt │ │ │ │
│ └──────────────┘ └────────────────────┘ └──────────────────┘ │
│ │
└──────────────────────────────────────────────────────────────────┘
```
### 2.2 Data Access Path
```
Compute thread accessing data:
┌─────────────────────────────────────────────────────────┐
│ Read-only path: L4.Map(originKey) │
│ ├─ StreamTransparent + no repack needed? │
│ │ └─ Yes → return original pointer directly │
│ │ (zero overhead) │
│ │ │
│ └─ CacheBuffer mode │
│ ├─ Swiss Table lookup (2.6ns, lock-free) │
│ ├─ Hit → update RRPV=0, return CacheLine handle │
│ └─ Miss → SpinWait for Predictor prefetch │
├─────────────────────────────────────────────────────────┤
│ Writable path: L4.MapWritable(originKey, preprocess, writeback) │
│ ├─ Hit → return existing partial sum cache line │
│ └─ Miss → allocate zero-initialized cache line, │
│ register writeback callbacks │
│ cache line marked dirty (dirty=1) │
│ eviction invokes preprocess + writeback │
├─────────────────────────────────────────────────────────┤
│ After computation: │
│ 3. CacheLine.Dispose() (refCount--) │
│ 4. L4.StepComplete() │
│ ├─ Predictor.AdvanceExecutionPoint() │
│ └─ Predictor.NeedsPrefetch? │
│ └─ Yes → CAS-acquire lock, execute prefetch │
└─────────────────────────────────────────────────────────┘
```
### 2.3 Key Parameters
| Parameter | Value | Description |
|------------------------|--------------------------|------------------------------------------------------|
| Total cache capacity | 64 MB | Per-NUMA-node contiguous memory |
| Cache line size | 8192 Bytes | = 64 × `double MAT_KERNEL` |
| Cache line count | 8192 | 64MB / 8192B |
| Mapping table type | Swiss Table | 1024 buckets × 16 slots, SIMD-accelerated |
| Cache line pool | Slab Allocator | Non-concurrent, fixed capacity, no auto-expansion |
| Eviction policy | RRPV mask + windowed OPT | SoA layout + SSE2 batch scan |
| Prefetch policy | Predictor windowed OPT | 2048-entry buffer, 1024-step lookahead |
| Writeback policy | Callback-driven | Invokes registered callbacks on dirty eviction/flush |
---
## 3. Key Components
### 3.1 Swiss Table (Pointer Mapping Table)
**Responsibility**: Maintains the `original pointer → cache pointer` mapping.
- Key: original pointer `nint`, pointing to data allocated by HLSF/malloc on a remote node
- Value: cache pointer `nint`, pointing to locally Slab-allocated cache line
- Hash: hardware `CRC32C` instruction on the 64-bit pointer
- Low 10 bits → bucket index (1024 buckets)
- Bits 10-16 → H2 control byte (used for SIMD pre-filtering)
- Scan: SSE2 `pmovmskb` instruction for parallel comparison of 16 control bytes
- Tombstone mechanism: deleted entries set ctrl to `0xFE`, preserving probe chain integrity
- Capacity: 1024 × 16 = 16,384 mappings, significantly larger than total cache line count (8192 lines),
ensuring extremely low hash collision rate
**Measured Performance** (Ryzen 7 6800H, .NET 8, BenchmarkDotNet):
| Operation | Swiss Table | Dictionary | Speedup |
|--------------------------|-------------|-------------|---------|
| Zipfian Lookup (8192×) | 20.0 μs | 76.1 μs | 3.8× |
| Uniform Lookup (8192×) | 18.6 μs | 50.6 μs | 2.7× |
| Insert (4096 entries) | 18.5 μs | 46.5 μs | 2.5× |
| SteadyState (4096 ops) | 25.3 μs | 68.3 μs | 2.7× |
Single lookup latency ≈ 2.6 ns, GC allocation = 0.
### 3.2 Slab Allocator (Cache Line Pool)
**Responsibility**: Manages allocation and reclamation of fixed-size (8192B) cache lines.
- Mode: **Non-concurrent, fixed capacity** (`FixedSlabAllocator<T>` mode),
reusing the existing `SlabAllocator.FromBuffer<T>()` interface
- No auto-expansion: capacity is fixed at initialization (64MB / 8192B = 8192 lines);
allocation failure triggers eviction rather than expansion
- Allocation complexity: O(1), pops from free stack
- Reclamation complexity: O(1), pushes back to free stack
- Alignment: all cache lines are 64B aligned, ensuring no cross-hardware-cache-line boundaries
### 3.3 Eviction Metadata (SoA Layout + RRPV Mask + Writeback Callbacks)
**Responsibility**: Tracks per-cache-line reference state, heat, and writeback information to determine eviction priority.
Replaces the four-level circular linked list from v0.1 with parallel arrays in Structure-of-Arrays layout:
```
SoA Layout — Parallel Arrays
_refFlags[i] : byte Reference count snapshot (0=free, non-zero=held)
_evictionFlags[i] : byte dirty(1bit) + RRPV(2bit) + costClass(1bit)
_originKeys[i] : nuint Original pointer (reverse-deregisters Swiss Table entry on eviction)
_preprocess[i] : nint Writeback pre-processing function pointer (called before dirty eviction)
_writeback[i] : nint Writeback function pointer (called during dirty eviction)
_callbackCtx[i] : nuint Callback context (passed to preprocess/writeback)
```
**Eviction Flag Bit Layout** (`_evictionFlags[i]`):
The RRPV array uses byte storage for each cache line's usage state; the low 2 bits encode the RRPV value.
**RRPV Semantics**:
| RRPV | Meaning | Eviction Priority |
|------|---------------------------------|---------------------------|
| 0 | Will be re-accessed very soon | Lowest (do not evict) |
| 1 | Will be accessed shortly | Low |
| 2 | Will be accessed in a while | Medium |
| 3 | May never be accessed again | Highest (evict first) |
**RRPV Update Events**:
| Event | Operation | Description |
|--------------------|---------------------------------|------------------------------------------------------|
| Load (insert) | RRPV = 0 | Freshly loaded data will be used soon |
| Hit | `_evictionFlags[i] = 0` | Bits 1-2 cleared, RRPV reset to 0 |
| Eviction scan | Find lines with RRPV=3 to evict | If none found, increment all RRPV +1 (aging) |
**Advantages over the v0.1 four-level circular linked list**:
| Comparison | v0.1 Circular List | v0.3 RRPV Mask + Callbacks |
|----------------------|-------------------------------------|--------------------------------------|
| Data structure | 4 doubly-linked lists + next ptrs | Byte array + function pointer arrays |
| Warm-up operation | Unlink + relink (4 pointer writes) | `+= 1` (1-byte write) |
| Aging operation | Per-node list migration | SIMD batch +1 (16 entries/cycle) |
| Eviction scan | Iterate list head (pointer chasing) | SIMD pattern matching (16/cycle) |
| Cache friendliness | Poor (pointer chasing) | Good (contiguous memory scan) |
| Reference protection | None | Lines with refCount > 0 not evicted |
| Writeback support | None | Callback-function driven |
### 3.4 Writeback Callback Mechanism
**Responsibility**: Provides flexible writeback strategies for writable cache lines, using native function pointers to avoid managed overhead.
**Design Principle**: All writable cache lines are treated as dirty by default. On eviction or explicit flush,
L4 itself does not manage the writeback strategy; instead, the bound predictor is responsible for implementing it.
The predictor is defined via the following interface:
```
public interface IPrefetchPredict
{
int Predict(
Span<CacheLinePrediction> node
);
bool Prefetch(
nuint origin,
ulong policy,
nuint cache
);
void WriteBack(
nuint origin,
ulong policy,
nuint cache
);
}
```
**Typical Callback Implementations**:
```
Scenario 1: Segment accumulation (FMA partial sum)
preprocess = null (no pre-processing needed; data is already a double array)
writeback = AccumulateAdd (element-wise dst[i] += src[i], AVX2-vectorized)
Scenario 2: Writeback of packed intermediate results
preprocess = UnpackKernelRow (reverse-repack from Pack format to row-major order)
writeback = AccumulateAdd
Scenario 3: PLU decomposition intermediate data
preprocess = null
writeback = DirectCopy (direct overwrite memcpy)
```
### 3.5 CacheLine (Cache Line Handle)
**Responsibility**: A cache line access handle held by the compute thread; prevents in-use lines from being evicted via reference counting.
```csharp
public ref struct CacheLine : IDisposable
{
private ref int _activeCount; // L4 global active count
private ref int _refCount; // Per-line reference count
private nuint _handle; // Local cache data address
// Construction: Interlocked.Increment(ref _activeCount)
// Interlocked.Increment(ref _refCount)
// Dispose: Interlocked.Decrement on both counters
}
```
- `ref struct` ensures it cannot escape to the heap
- Reference counts use `Interlocked` atomic operations for thread safety
- Lines with `_refCount > 0` are skipped during eviction scans
- StreamTransparent mode returns a CacheLine that does not occupy a cache slot
- Both read-only and writable paths return the same CacheLine type,
callers decide read/write behavior via `.Address`
### 3.6 Predictor (Prefetch Predictor)
**Responsibility**: Provides deterministic prefetch predictions for L4 based on the windowed OPT strategy.
**Core Principle**: Matrix computation task sequences are deterministic — the scheduler pre-allocates
task blocks `(kr, kc, zz)` for each thread, allowing precise computation of the cache line keys
required at each future step. This gives L4 a predictive capability that hardware caches cannot
possess: **direct visibility into the future**, rather than inferring the future from history as
strategies like Hawkeye do.
```
Predictor Sliding Window:
Task sequence: ... [T_now] [T+1] [T+2] ... [T+1023] ... [T+2047]
↑ execution point ↑ buffer tail
Window capacity = 2048 prediction entries
Prefetch trigger threshold = trigger refill when prefetch frontier
leads execution point by fewer than 1024 steps
Refill batch size = 64 entries (avoids excessive single-refill latency)
```
**数据结构**:
```csharp
internal class Predictor
{
private const int WindowCapacity = 2048;
private const int RefillThreshold = 1024;
private PrefetchEntry[] _buffer; // Circular prediction buffer
private int _head; // Prefetch frontier (prefetched up to here)
private int _tail; // Production position (scheduler fills up to here)
private int _executed; // Execution point (compute thread has completed up to here)
private HashSet<nuint> _activeKeys; // All active keys within the window (deduplicated)
}
struct PrefetchEntry
{
public nuint Key; // Cache line identifier
public nuint SourceAddress; // Source data address
public bool NeedsRepack; // Whether non-contiguous repacking is needed
}
```
**Cooperation with Eviction Policy**:
```
Eviction priority (from highest to lowest priority):
1. refCount==0, not in window, RRPV==3, dirty==0 ← First (zero-cost eviction)
2. refCount==0, not in window, RRPV==3, dirty==1 ← Second (needs writeback but cold)
3. refCount==0, not in window, RRPV<3, dirty==0 ← Ordered by RRPV
4. refCount==0, not in window, RRPV<3, dirty==1 ← Ordered by RRPV
5. refCount==0, in window ← Avoid evicting if possible
6. refCount>0 ← Never evict
```
Windowed OPT means most eviction decisions do not depend on RRPV — "not in window" alone is
sufficient to determine evictability. RRPV provides fine-grained differentiation among
out-of-window lines; the dirty bit preferentially evicts clean lines at equal RRPV to avoid writeback overhead.
### 3.7 Concurrency Model
L4 uses a **no-extra-threads** design; all operations are performed by compute threads themselves:
| Operation | Executor | Concurrency |
|------------------------------------|---------------------------------|--------------------------------------|
| Lookup (Swiss Table read) | Any compute thread | ✅ Fully parallel, lock-free |
| HIT RRPV update | Any compute thread | ✅ Single-byte write, no sync needed |
| refCount increment/decrement | Any compute thread | ✅ Interlocked atomic operations |
| Prefetch refresh (eviction + load) | CAS-winner thread | Serial (only one thread enters) |
| Dirty writeback | CAS-winner thread | Serial (within eviction flow) |
| FlushAllDirty | Single thread after computation | Serial (no concurrency) |
**Prefetch Refresh CAS Lock**:
```csharp
private int _refreshLock; // 0=free, 1=locked
private void TryRefreshPrefetch()
{
if (Interlocked.CompareExchange(ref _refreshLock, 1, 0) != 0)
return; // Another thread is doing it; current thread continues computing
try { /* eviction + prefetch */ }
finally { Volatile.Write(ref _refreshLock, 0); }
}
```
**Swiss Table Concurrent Read Safety Guarantee**: The prefetch refresh thread follows strict write ordering
on insertion: write Slab data first, then write eviction metadata and callback pointers, `sfence`, then
write the Swiss Table control byte and key/value, ensuring that when a compute thread reads the control
byte, the data is already fully visible.
---
## 4. Operation Flows
### 4.1 Cache Lookup (Read-Only Hit)
1. Compute CRC32C hash of the original pointer
2. Swiss Table lookup: SIMD scan ctrl → compare key → return cache pointer
3. Update RRPV: `_evictionFlags[slot] &= 0xF9` (RRPV zeroed, dirty bit unchanged)
4. Construct CacheLine handle (refCount++)
5. Return CacheLine; upper-layer code accesses local data via `.Address`
### 4.2 Cache Lookup (Writable Hit)
1. Swiss Table lookup hits
2. Update RRPV: `_evictionFlags[slot] &= 0xF9`
3. Construct CacheLine handle (refCount++)
4. Return CacheLine; upper-layer code reads/writes the cache line via `.Address` (e.g., partial sum +=)
### 4.3 Cache Miss (Read-Only)
1. Swiss Table lookup misses
2. Compute thread enters `Thread.Sleep(0)` spin loop
3. Wait for Predictor-driven prefetch refresh to load the line into L4
4. Exit wait loop when Swiss Table shows a hit
5. Construct CacheLine handle and return
Under normal operation, the Predictor always leads the execution point by 1024+ steps,
misses occur only during cold start or extreme bursts.
### 4.4 Cache Miss (Writable)
1. Swiss Table lookup misses
2. Allocate a cache line from Slab
3. If Slab has no free slots → trigger eviction (see 4.6)
4. Zero-initialize the cache line (`Unsafe.InitBlock(dest, 0, 8192)`)
5. Register writeback callbacks: `_preprocess[slot]`, `_writeback[slot]`, `_callbackCtx[slot]`
6. Set eviction flags: `dirty=1, RRPV=0`
7. Write `_originKeys[slot]`
8. `sfence`
9. Write Swiss Table mapping
10. Construct CacheLine handle and return
### 4.5 Prefetch Refresh
Triggered by compute threads in `StepComplete()`; CAS lock ensures single-thread execution:
1. Synchronize refFlags snapshot (`_refCounts[i] > 0 → _refFlags[i] = 1`)
2. Retrieve pending prefetch entries from Predictor (up to 64 at a time)
3. For each entry:
- Swiss Table hit → mark RRPV=0 (hot-line protection)
- Swiss Table miss → execute load:
a. Allocate cache line from Slab
b. If Slab has no free slots → trigger eviction (see 4.6)
c. Execute Pack reorder or memcpy based on `NeedsRepack` flag
d. Write eviction metadata (originKey, RRPV=0, dirty=0, callbacks=null)
e. `sfence`
f. Write Swiss Table mapping
### 4.6 Cache Eviction (SIMD Scan + Writeback)
```
Eviction scan pseudocode:
Pass 1: Find refCount==0, RRPV==3, dirty==0 (zero-cost eviction)
for i in range(scanCursor, CacheLineCount), step=16:
refChunk = SSE2.Load(refFlags + i)
refFreeMask = SSE2.MoveMask(SSE2.CompareEqual(refChunk, zero))
evChunk = SSE2.Load(evictionFlags + i)
// RRPV==3 and dirty==0 → byte & 0x07 == 0x06
masked = SSE2.And(evChunk, 0x07)
coldCleanMask = SSE2.MoveMask(SSE2.CompareEqual(masked, 0x06))
candidates = refFreeMask & coldCleanMask
while candidates != 0:
bit = TrailingZeroCount(candidates)
idx = i + bit
if originKeys[idx] not in Predictor.ActiveKeys:
evict(idx) // No writeback, discard directly
return
candidates &= candidates - 1
Pass 2: Find refCount==0, RRPV==3, dirty==1 (needs writeback)
Same as above, but match byte & 0x07 == 0x07
evict triggers writeback callbacks
Pass 3: Increment all RRPV+1 (aging), retry
Up to 3 rounds
```
**逐出执行**:
```csharp
private void Evict(int slot)
{
byte flags = _evictionFlags[slot];
// Dirty writeback
if ((flags & 0x01) != 0)
{
nuint origin = _originKeys[slot];
nuint cached = SlotAddressFromIndex(slot);
nuint ctx = _callbackCtx[slot];
// 1. Pre-processing (optional)
var preprocess = (delegate* unmanaged<nuint, nuint, void>)_preprocess[slot];
if (preprocess != null)
preprocess(cached, ctx);
// 2. Writeback
var writeback = (delegate* unmanaged<nuint, nuint, nuint, void>)_writeback[slot];
if (writeback != null)
writeback(origin, cached, ctx);
}
// Standard eviction
_map.TryRemove(_originKeys[slot], out _);
_slab.Free(SlotAddressFromIndex(slot));
_evictionFlags[slot] = 0;
_originKeys[slot] = 0;
_preprocess[slot] = 0;
_writeback[slot] = 0;
_callbackCtx[slot] = 0;
}
```
### 4.7 显式刷新
```csharp
/// <summary>
/// Explicitly writes back the specified dirty cache line. Called after all segments complete.
/// </summary>
public void FlushDirty(nuint originKey)
{
if (_map.TryGet(originKey, out nuint addr))
{
int slot = SlotIndex(addr);
if ((_evictionFlags[slot] & 0x01) != 0)
{
// Execute pre-processing + writeback
InvokeWriteBack(slot);
_evictionFlags[slot] &= 0xFE; // Clear dirty bit
}
}
}
/// <summary>
/// Writes back all dirty cache lines in bulk. Called after entire matrix computation completes.
/// </summary>
public void FlushAllDirty()
{
for (int i = 0; i < CacheLineCount; i++)
{
if ((_evictionFlags[i] & 0x01) != 0 && _originKeys[i] != 0)
{
InvokeWriteBack(i);
_evictionFlags[i] &= 0xFE;
}
}
}
```
### 4.8 Working Mode Switch
```csharp
public enum L4Mode
{
/// <summary>
/// Pass-through mode: Map() returns the original pointer directly; L4 does not intervene.
/// Suitable for single-NUMA + no-repack scenarios (e.g., vector addition, scalar multiplication).
/// </summary>
StreamTransparent,
/// <summary>
/// Buffer mode: full lookup/Pack/cache/eviction pipeline.
/// Suitable for FMA scenarios requiring repacking or multi-NUMA scenarios.
/// </summary>
CacheBuffer
}
```
Mode is determined at construction time and does not change at runtime. Decision logic:
- Requires Pack reordering (FMA core rows of matrix A) → CacheBuffer
- Multiple NUMA nodes → CacheBuffer
- Requires segment accumulation → CacheBuffer
- Single NUMA + no reordering + no segment → StreamTransparent
---
## 5. Segment Accumulation Scenario
### 5.1 Problem
For matrix multiplication `C = A × B` when the K dimension is large, segmented accumulation is required:
```
segment 0: C_partial = A[kr, 0..KC] × B[0..KC, kc]
segment 1: C_partial += A[kr, KC..2KC] × B[KC..2KC, kc]
...
segment N: C_partial += A[kr, N*KC..K] × B[N*KC..K, kc]
Final: C[kr, kc] = Σ all segment partial sums
```
If each segment's result is `+=` directly to the C matrix,
and C resides on a remote NUMA node, every segment incurs cross-node write latency.
### 5.2 L4 Solution
Temporarily store C's partial sums in L4's writable cache lines:
```
segment 0: c = l4.MapWritable(C_key, null, &AccumulateAdd)
// First miss → allocate zero-initialized cache line
FMA(c.Address, a.Address, b.Address) // c += a*b
c.Dispose()
segment 1: c = l4.MapWritable(C_key, null, &AccumulateAdd)
// Hit! Returns existing partial sum cache line
FMA(c.Address, a.Address, b.Address) // c += a*b
c.Dispose()
... after all segments complete ...
l4.FlushDirty(C_key)
// Calls AccumulateAdd(C_origin, cacheline, ctx)
// → Adds local partial sum back to remote C matrix in one shot
```
### 5.3 Benefits
```
Without segment buffering (OpenBLAS approach):
Each segment writes C matrix across NUMA
Remote writes per output tile = Zk times
Per-write latency = ~200-300ns (cross-node)
With L4 segment buffering:
All segment += operations complete locally in L4 cache lines
Partial sums likely remain in L1/L2 (8KB << L1 capacity)
Remote writes per output tile = 1 (final FlushDirty)
Writeback uses AVX2-vectorized stores: 8192B = 32 × 256-bit stores
Reduction ratio = Zk : 1
```
---
## 6. Uniqueness and Advantages
### 6.1 Deterministic Prefetch (Windowed OPT)
Essential difference from hardware cache replacement policies (LRU, RRIP, Hawkeye):
| Policy | Information Source | Prediction Method | Accuracy |
|---------------------|-------------------------|-------------------------------|---------------------------|
| LRU | Past access order | Least recently used | Moderate |
| RRIP | Insert/hit markers | Predict reuse distance | Moderate-High |
| Hawkeye | OPTgen retrospection | Learn past to infer future | High |
| **L4 Predictor** | **Future task queue** | **Direct future visibility** | **Perfect** (within window) |
L4's Predictor is a **deterministic Oracle** — it does not predict; it knows.
Because matrix computation task sequences are fully determined at scheduler allocation time.
### 6.2 NUMA Adaptability
The same code automatically gains NUMA benefits on different hardware configurations:
| Hardware Configuration | L4 Behavior |
|-------------------------|-------------------------------------------------------------|
| Single-socket CPU | Pack cache + segment accumulation buffer |
| Dual-socket CPU | Pack cache + NUMA migration cache + segment accum. |
| Hybrid-architecture CPU | Pack cache + cross-domain migration cache + segment accum. |
### 6.3 Callback-Driven Writeback Strategy
Using native function pointers for writeback avoids:
- Virtual method call overhead
- Interface dispatch overhead
- Delegate GC pressure
- Hard-coding specific writeback logic
Callers register `preprocess` and `writeback` callbacks at `MapWritable` time,
L4 invokes them directly through function pointers on eviction or Flush,
**zero managed overhead, zero virtual dispatch, performance equivalent to C function calls**.
### 6.4 Reference Count Protection
Through `CacheLine` handle reference counting, cache lines actively in use by compute threads are
absolutely never evicted (including writeback not being triggered). The SIMD eviction scan skips
lines with `refCount > 0` immediately via the `_refFlags` array, without checking linked lists or acquiring locks.
### 6.5 Cache Line Size Aligned to Matrix Kernel
The 8192B cache line size is not arbitrary:
- 1 `double MAT_KERNEL` (4×4 double matrix) = 128 Bytes
- 1 cache line = 64 `MAT_KERNEL` units
- Matrix multiplication micro-kernels operate on data sizes perfectly aligned to cache lines
- Compute kernel functions operate on cache lines as units; no cross-cache-line accesses occur
### 6.6 Fixed-Capacity Determinism
- Slab allocator never expands → memory footprint upper bound is known (64 MB)
- SIMD eviction scan → latency upper bound is known (at most 3 aging rounds × 512 SIMD comparisons)
- Swiss Table fixed 16,384 slots → lookup latency upper bound is known
- Writeback callback latency is controlled by the caller → predictable
- Worst-case L4 latency = lookup + eviction (incl. writeback) + allocation + memcpy, all statically computable
### 6.7 Reuse of Existing StgSharp Infrastructure
| L4 Component | Reused StgSharp Infrastructure |
|-----------------------|-------------------------------------------------------|
| Mapping table | `SwissTable` (implemented and benchmark-validated) |
| Mapping table SIMD | `IBucketScanner` + `SIMDID` architecture detection |
| Cache line alloc | `SlabAllocator.FromBuffer<T>()` |
| Hash computation | Hardware `CRC32C` instruction |
| Memory allocation | `NativeMemory.AlignedAlloc` |
| Future multi-platform | `IBucketScanner` + NRGA compilation pipeline |
### 6.8 Originality Analysis
After surveying existing mainstream numerical computation libraries, L4's design has
**no direct precedent or equivalent in known public implementations**.
#### NUMA Strategy Comparison with Existing Libraries
| Library / Framework | NUMA Strategy | Software Cache Layer | Mapping Table | Eviction Policy | Dirty Writeback |
|---------------------|-------------------------------------|---------------------|-----------------|-------------------------|--------------------|
| Intel MKL | Thread pinning + `numactl` | None | None | None | None |
| OpenBLAS | Thread affinity + block alignment | None | None | None | None |
| BLIS | Multi-level block alignment L1/L2/L3| None | None | None | None |
| Eigen | No explicit NUMA handling | None | None | None | None |
| cuBLAS / rocBLAS | GPU memory (hardware-solved) | None | None | None | None |
| **StgSharp L4** | **Software-managed thread-local cache** | **Yes** | **Swiss Table** | **RRPV + windowed OPT** | **Callback-driven** |
---
## 7. 从 v0.1 到 v0.3 的变更摘要
| 项目 | v0.1 | v0.3 |
|-----------------|-------------------------------|---------------------------------------|
| 逐出数据结构 | 4 级环形双向链表 | RRPV 2-bit 掩码 + 8KB SoA 数组 |
| 逐出扫描方式 | 链表头指针遍历 | SSE2 批量模式匹配(16 行/周期) |
| 热���更新 | 链表节点摘插(4 指针写) | 单字节位运算 `&= 0x03` |
| 老化操作 | 逐节点移链表 | SIMD 批量 +1 |
| 引用保护 | 无 | refCount > 0 时 SIMD 扫描跳过 |
| 预取策略 | 无 | Predictor 窗口 OPT(2048 条缓冲) |
| 线程模型 | L4 服务线程 + Command Queue | 无额外线程,CAS 抢锁 |
| 命令队列 | 链式 CommandCache 分段 | 不需要(预取刷新替代命令驱动) |
| 缓存行元数据 | Slab 分配的 CacheLineHead 结构 | SoA 平行数组 + 函数指针数组 |
| 工作模式 | 单一模式 | StreamTransparent / CacheBuffer 切换 |
| 写回策略 | 无(只读缓存) | dirty 位 + preprocess/writeback 回调 |
| 写回机制 | 无 | 本机函数指针,零托管开销 |
---
## 8. 未来工作
- [ ] L4 根类型重构(组合 Swiss Table、Slab、SoA 元数据、Predictor)
- [ ] Predictor 与矩阵乘法调度器的集成
- [ ] Segment 累加写回的 AVX2 向量化实现
- [ ] SIMD 逐出扫描的 AVX2 升级(32 行/周期)
- [ ] Swiss Table ARM NEON 实现
- [ ] Swiss Table AVX-512 升级(64 槽扫描从 16 次降至 4 次)
- [ ] 墓碑累积监控与自动 rehash 策略
- [ ] 性能基准测试(NUMA 本地 vs 远端访问延迟对比)
- [ ] Segment 累加 vs 直接写回的 benchmark 对比
- [ ] 与逐线程矩阵上下文的集成验证
- [ ] 双路 6242R 环境下的 NUMA 搬运性能验证
- [ ] 远期:PPC64LE 平台的 VSX 原生 Swiss Table 扫描支持
---
## 9. 参考
- Google Abseil Swiss Tables Design Notes
- HP Labs RRIP (Re-Reference Interval Prediction) 论文
- Hawkeye: Efficient Cache Replacement via OPTgen (ISCA 2017)
- StgSharp HLSF Allocator 设计文档
- StgSharp SIMD Architecture (`SIMDID` 体系)
- StgSharp NRGA Roadmap
- StgSharp `SlabAllocator<T>` 实现
- StgSharp `SwissTable` 实现及 Benchmark
@@ -0,0 +1,364 @@
// -----------------------------------------------------------------------------
// file="Ptr64"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
#pragma warning disable CA2225
public readonly struct Ptr64 : IEquatable<Ptr64>, IComparable<Ptr64>, IFormattable
{
public Ptr64(
ulong value
)
{
Value = value;
}
public ulong Value { get; }
public int CompareTo(
Ptr64 other
)
{
return Value.CompareTo(other.Value);
}
public bool Equals(
Ptr64 other
)
{
return Value == other.Value;
}
public override bool Equals(
object? obj
)
{
return obj is Ptr64 other && Equals(other);
}
public override int GetHashCode()
{
return Value.GetHashCode();
}
public override string ToString()
{
return Value.ToString();
}
public string ToString(
string? format,
IFormatProvider? formatProvider
)
{
return Value.ToString(format, formatProvider);
}
public static Ptr64 operator -(
Ptr64 left,
ulong right
)
{
return new(left.Value - right);
}
public static ulong operator -(
Ptr64 left,
Ptr64 right
)
{
return left.Value - right.Value;
}
public static bool operator !=(
Ptr64 left,
Ptr64 right
)
{
return left.Value != right.Value;
}
public static Ptr64 operator &(
Ptr64 left,
ulong mask
)
{
return new(left.Value & mask);
}
public static Ptr64 operator |(
Ptr64 left,
ulong mask
)
{
return new(left.Value | mask);
}
public static Ptr64 operator +(
Ptr64 left,
ulong right
)
{
return new(left.Value + right);
}
public static bool operator <(
Ptr64 left,
Ptr64 right
)
{
return left.Value < right.Value;
}
public static Ptr64 operator <<(
Ptr64 left,
int shift
)
{
return new Ptr64(left.Value << shift);
}
public static bool operator <=(
Ptr64 left,
Ptr64 right
)
{
return left.Value <= right.Value;
}
public static bool operator ==(
Ptr64 left,
Ptr64 right
)
{
return left.Value == right.Value;
}
public static bool operator >(
Ptr64 left,
Ptr64 right
)
{
return left.Value > right.Value;
}
public static bool operator >=(
Ptr64 left,
Ptr64 right
)
{
return left.Value >= right.Value;
}
public static Ptr64 operator >>(
Ptr64 left,
int shift
)
{
return new Ptr64(left.Value >> shift);
}
public static implicit operator Ptr64(
ulong value
)
{
return new(value);
}
public static implicit operator ulong(
Ptr64 value
)
{
return value.Value;
}
}
public readonly struct SPtr64 : IEquatable<SPtr64>, IComparable<SPtr64>, IFormattable
{
public SPtr64(
long value
)
{
Value = value;
}
public long Value { get; }
public int CompareTo(
SPtr64 other
)
{
return Value.CompareTo(other.Value);
}
public bool Equals(
SPtr64 other
)
{
return Value == other.Value;
}
public override bool Equals(
object? obj
)
{
return obj is SPtr64 other && Equals(other);
}
public override int GetHashCode()
{
return Value.GetHashCode();
}
public override string ToString()
{
return Value.ToString();
}
public string ToString(
string? format,
IFormatProvider? formatProvider
)
{
return Value.ToString(format, formatProvider);
}
public static SPtr64 operator -(
SPtr64 left,
long right
)
{
return new(left.Value - right);
}
public static long operator -(
SPtr64 left,
SPtr64 right
)
{
return left.Value - right.Value;
}
public static bool operator !=(
SPtr64 left,
SPtr64 right
)
{
return left.Value != right.Value;
}
public static SPtr64 operator &(
SPtr64 left,
long mask
)
{
return new(left.Value & mask);
}
public static SPtr64 operator |(
SPtr64 left,
long mask
)
{
return new(left.Value | mask);
}
public static SPtr64 operator +(
SPtr64 left,
long right
)
{
return new(left.Value + right);
}
public static bool operator <(
SPtr64 left,
SPtr64 right
)
{
return left.Value < right.Value;
}
public static SPtr64 operator <<(
SPtr64 left,
int shift
)
{
return new SPtr64(left.Value << shift);
}
public static bool operator <=(
SPtr64 left,
SPtr64 right
)
{
return left.Value <= right.Value;
}
public static bool operator ==(
SPtr64 left,
SPtr64 right
)
{
return left.Value == right.Value;
}
public static bool operator >(
SPtr64 left,
SPtr64 right
)
{
return left.Value > right.Value;
}
public static bool operator >=(
SPtr64 left,
SPtr64 right
)
{
return left.Value >= right.Value;
}
public static SPtr64 operator >>(
SPtr64 left,
int shift
)
{
return new SPtr64(left.Value >> shift);
}
public static implicit operator long(
SPtr64 value
)
{
return value.Value;
}
public static implicit operator SPtr64(
long value
)
{
return new(value);
}
}
#pragma warning disable CA2225
}
@@ -0,0 +1,39 @@
// -----------------------------------------------------------------------------
// file="SlabAllocationHandle"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.HighPerformance.Memory
{
public unsafe struct SlabAllocationHandle<T> : IDisposable where T : unmanaged
{
private readonly SlabAllocator<T> _allocator;
internal SlabAllocationHandle(
SlabAllocator<T> allocator,
bool locked
)
{
_allocator = allocator;
Locked = locked;
}
public bool Locked { get; set; }
public void Dispose()
{
if (Locked)
{
_allocator.ExitBufferReading();
Locked = false;
}
}
}
}
@@ -0,0 +1,118 @@
// -----------------------------------------------------------------------------
// file="SlabAllocator.Chunked"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private bool _disposed;
private BufferStack<nuint> _buffers;
private BufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ChunkedSlabAllocator(
nuint count
)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * ((nuint)_elementSize));
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + ((nuint)(i * _elementSize));
}
_buffers = new(4);
_buffers.Push(_currentBuffer);
_recycle = BufferStackBuilder.Create(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
// nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _currentCapacity)
{
nuint cap = _currentCapacity;
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * ((nuint)_elementSize));
_buffers.Push(newBuffer);
_currentBuffer = newBuffer;
_currentIndex = 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _currentBuffer + (((nuint)_elementSize) * curIndex);
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
// NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer))
{
NativeMemory.Free((void*)buffer);
}
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(
nuint index
)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ChunkedSlabAllocator()
{
Dispose();
}
}
}
@@ -0,0 +1,142 @@
// -----------------------------------------------------------------------------
// file="SlabAllocator.ChunkedConcurrent"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T>
where T : unmanaged
{
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private ConcurrentBufferStack<nuint> _buffers;
private ConcurrentBufferStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _currentBuffer;
private nuint _currentCapacity, _currentIndex;
public ConcurrentChunkedSlabAllocator(
nuint count
)
{
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
_currentCapacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _currentBuffer + (nuint)(i * _elementSize);
}
_buffers = new(4);
_recycle = BufferStackBuilder.CreateConcurrent(span);
_currentCapacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
while (true)
{
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
nuint curIndex = Volatile.Read(ref _currentIndex);
_lock.EnterMetaDataRead();
try
{
if (curIndex >= _currentCapacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
nuint cap = Volatile.Read(ref _currentCapacity);
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
_buffers.Push(bufferHandle);
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
_ = Interlocked.Exchange(ref _currentIndex, 0);
}
finally
{
_lock.ExitExpansionProcess();
}
}
curIndex = Volatile.Read(ref _currentIndex);
_ = Interlocked.Add(
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
return _currentBuffer + ((nuint)_elementSize * curIndex);
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
NativeMemory.Free((void*)_currentBuffer);
while (_buffers.TryPop(out nuint buffer)) {
NativeMemory.Free((void*)buffer);
}
_lock.Dispose();
_buffers.Dispose();
_recycle.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(
nuint index
)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~ConcurrentChunkedSlabAllocator()
{
Dispose();
}
}
}
@@ -0,0 +1,111 @@
// -----------------------------------------------------------------------------
// file="SlabAllocator.Fixed"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.HighPerformance.Memory;
using StgSharp.Threading;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory
{
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
{
private Action<nuint> _freeHandle;
private bool _disposed;
private CapacityFixedStack<nuint> _recycle;
private readonly int _elementSize = Unsafe.SizeOf<T>();
private nuint _buffer;
private nuint _Capacity, _currentIndex;
internal FixedSlabAllocator(
nuint buffer,
nuint count,
Action<nuint> freeHandle
)
{
_freeHandle = freeHandle;
_buffer = buffer;
_Capacity = count;
int initCount = count switch
{
<= 4 => 4,
> 64 => 64,
_ => (int)count
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = _buffer + ((nuint)(i * _elementSize));
}
_recycle = CapacityFixedStackBuilder.Create(span);
_Capacity = count;
_currentIndex = (nuint)initCount;
}
public override nuint Allocate()
{
if (_recycle.TryPop(out nuint handle)) {
return handle;
}
// nuint bufferHandle = _currentBuffer;
nuint curIndex = _currentIndex;
if (curIndex >= _Capacity) {
return 0;
}
curIndex = _currentIndex;
_currentIndex += 1;
return _buffer + (((nuint)_elementSize) * curIndex);
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_freeHandle(_buffer);
_recycle = null!;
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading() { }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void Free(
nuint index
)
{
_recycle.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
return new SlabAllocationHandle<T>(this, false);
}
~FixedSlabAllocator()
{
Dispose();
}
}
}
@@ -0,0 +1,146 @@
// -----------------------------------------------------------------------------
// file="SlabAllocator.SequencialConcurrent"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.HighPerformance.Memory
{
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T>
where T : unmanaged
{
private T* _buffer;
private bool _disposed;
private readonly BufferExpansionLock _lock = new();
private readonly ConcurrentBufferStack<nuint> _stack;
private readonly int _elementSize;
private nuint _highestAllocated;
private ulong _capacity;
public ConcurrentSequentialSlabAllocator(
nuint initialCapacity
)
{
_elementSize = Unsafe.SizeOf<T>();
_capacity = initialCapacity;
_highestAllocated = 0;
_buffer = (T*)NativeMemory.Alloc(((nuint)_capacity) * ((nuint)_elementSize));
int initCount = _capacity switch
{
<= 4 => 4,
> 64 => 64,
_ => ((int)_capacity) / 4
};
Span<nuint> span = stackalloc nuint[initCount];
for (int i = 0; i < initCount; i++) {
span[i] = (nuint)i;
}
_stack = BufferStackBuilder.CreateConcurrent(span);
_highestAllocated += (nuint)initCount;
}
internal T* BasePointer => _buffer;
public override nuint Allocate()
{
if (_stack.TryPop(out nuint index)) {
return index;
}
while (true)
{
index = Volatile.Read(ref _highestAllocated);
_lock.EnterMetaDataRead();
try
{
if (index >= _capacity)
{
_lock.ExitMetaDataRead();
_lock.EnterExpansionProcess();
try
{
_capacity = Volatile.Read(ref _capacity) * 2;
nuint newSize = ((nuint)_capacity) * ((nuint)_elementSize);
_lock.EnterBufferCopy();
try
{
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
}
finally
{
_lock.ExitBufferCopy();
}
}
finally
{
_lock.ExitExpansionProcess();
}
}
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
return index;
}
finally
{
if (_lock.IsThreadReadingMetaData) {
_lock.ExitMetaDataRead();
}
}
}
}
public override void Dispose()
{
if (_disposed) {
return;
}
_disposed = true;
_stack.Dispose();
NativeMemory.Free(_buffer);
_lock.Dispose();
GC.SuppressFinalize(this);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void EnterBufferReading()
{
_lock.EnterBufferRead();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override void ExitBufferReading()
{
_lock.ExitBufferRead();
}
public override void Free(
nuint index
)
{
_stack.Push(index);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override SlabAllocationHandle<T> ReadAllocation()
{
EnterBufferReading();
return new SlabAllocationHandle<T>(this, true);
}
~ConcurrentSequentialSlabAllocator()
{
Dispose();
}
}
}
@@ -0,0 +1,115 @@
// -----------------------------------------------------------------------------
// file="SlabAllocator"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
namespace StgSharp.HighPerformance.Memory
{
public static class SlabAllocator
{
/// <summary>
/// Creates a new SLAB allocator instance with the specified configuration.
/// </summary>
/// <param name="count">
/// Initial number of objects that can be allocated.
/// </param>
/// <param name="layout">
/// Memory layout strategy for the SLAB allocator.
/// </param>
/// <param name="concurrentSupport">
/// Whether to enable concurrent access support.
/// </param>
/// <returns>
/// A new SLAB allocator instance.
/// </returns>
/// <exception cref="ArgumentOutOfRangeException">
/// Thrown when an invalid layout is specified.
/// </exception>
/// <exception cref="NotSupportedException">
/// Thrown when requesting unsupported configurations.
/// </exception>
/// <remarks>
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
/// allocators themselves may be designed for single-threaded use, allocated objects are
/// frequently shared across multiple threads in real-world applications. Due to the
/// special implementation of expandable concurrent sequential SLAB allocators, they rely
/// on internal expansion locks to ensure that memory buffer expansions (which involve
/// copying the entire buffer to a new location) do not occur while other threads are
/// reading from allocated regions. </para> <para><strong> Sequential Layout Limitations:
/// </strong><br /> Sequential layout SLAB allocators cannot provide a single- threaded
/// version because:<list type="bullet"><item><description> They use contiguous memory
/// layout that requires buffer expansion when capacity is exceeded
/// </description></item><item><description> Buffer expansion involves copying all
/// existing data to a new memory location </description></item><item><description>Without
/// proper synchronization, concurrent access to allocated objects during synchronization,
/// concurrent access to allocated objects during expansion would cause memory
/// corruption</description></item><item><description>The expansion lock mechanism is
/// essential even if allocation itself is single- threaded
/// </description></item></list></para> <para><strong> Chunked Layout: </strong><br />
/// Chunked layout allocators can optionally provide single-threaded versions since they
/// don't require buffer expansion and copying. Each chunk is allocated independently,
/// avoiding the memory relocation issues present in sequential layouts. </para>
/// </remarks>
public static SlabAllocator<T> Create<T>(
nuint count,
SlabBufferLayout layout,
bool concurrentSupport = true
) where T : unmanaged
{
return layout switch
{
SlabBufferLayout.Sequential => concurrentSupport ?
new ConcurrentSequentialSlabAllocator<T>(count) :
throw (new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
SlabBufferLayout.Chunked => concurrentSupport ?
new ConcurrentChunkedSlabAllocator<T>(count) :
new ChunkedSlabAllocator<T>(count),
_ => throw (new ArgumentOutOfRangeException(nameof(layout), layout, null))
};
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static SlabAllocator<T> FromBuffer<T>(
nuint buffer,
nuint count,
Action<nuint> freeHandle
) where T : unmanaged
{
return new FixedSlabAllocator<T>(buffer, count, freeHandle);
}
}
public abstract class SlabAllocator<T> : IDisposable where T : unmanaged
{
public abstract nuint Allocate();
public abstract void Dispose();
public abstract void EnterBufferReading();
public abstract void ExitBufferReading();
public abstract void Free(
nuint index
);
public abstract SlabAllocationHandle<T> ReadAllocation();
}
public enum SlabBufferLayout
{
Sequential,
Chunked
}
}
@@ -0,0 +1,669 @@
// -----------------------------------------------------------------------------
// file="TwoLayerSegregatedFitAllocator"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Drawing;
using System.Linq;
using System.Numerics;
using System.Reflection.Metadata;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using static StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
namespace StgSharp.HighPerformance.Memory
{
public unsafe class TwoLayerSegregatedFitAllocator : IDisposable
{
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
private readonly int[] _bucket;
private readonly Metadata* _spareMemory;
private readonly Action<nuint> _freeHandle;
private bool disposedValue;
private readonly int _align;
private readonly int _maxLevel;
private int _metadataEnd;
private readonly Ptr64 _basePtr;
private readonly ulong _maxAllocSize;
private readonly ulong _totalSize;
internal TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
ulong maxBlockSize,
Func<nuint, int, nuint> allocator,
Action<nuint> freeHandle
)
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle) { }
internal TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
ulong maxBlockSize,
void* externalPointer,
Action<nuint> freeHandle
)
{
_totalSize = byteSize;
_basePtr = (Ptr64)(ulong)externalPointer;
_align = align;
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
_maxLevel = int.MaxValue;
_maxLevel = GetLevel(_maxAllocSize);
_bucket = new int[GetLevel(_maxAllocSize) + 2];
_spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
_freeHandle = freeHandle ?? (static_ => { });
_metadataEnd = 0;
// init spare memory node
_spareMemory->Position = _basePtr;
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
_spareMemory->NextLevel = 0;
_spareMemory->NextNear = 0;
_spareMemory->PreviousLevel = 0;
_spareMemory->PreviousNear = 0;
}
public TwoLayerSegregatedFitAllocator(
nuint byteSize
)
: this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
uint maxBlockSize
)
: this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align
)
: this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public TwoLayerSegregatedFitAllocator(
nuint byteSize,
int align,
nuint maxBlockSize
)
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
{
if (disposing) {
_freeHandle((nuint)_basePtr);
}
disposedValue = true;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private int GetLevel(
ulong size
)
{
int origin = 57 - BitOperations.LeadingZeroCount(size);
return int.Min(origin, _maxLevel);
}
/// <summary>
/// Put a metadata node back to free list and waiting for next allocation.
/// </summary>
/// <param name="node">
///
/// </param>
private void RecycleNode(
Metadata* node
)
{
int nodeIdx = (int)(_spareMemory - node);
int level = GetLevel(node->Size);
int nextLevelIdx = node->NextLevel;
int prevLevelIdx = node->PreviousLevel;
RemoveFromBucket(level, node, nodeIdx);
// remove from position chain
int prevNearIdx = node->PreviousNear;
int nextNearIdx = node->NextNear;
Metadata* prevNear = &_spareMemory[-prevNearIdx];
prevNear->NextNear = nextNearIdx;
if (nextNearIdx != 0)
{
Metadata* nextNear = &_spareMemory[-nextNearIdx];
nextNear->PreviousNear = prevNearIdx;
} else
{
_spareMemory->PreviousNear = prevNearIdx;
}
node->PreviousNear = 0;
node->NextNear = 0;
node->Position = 0;
node->SizeForSpare = 0;
node->State = AllocationState.SpareOrCollect;
node->PreviousLevel = 0;
int nextIdx = _spareMemory->NextLevel;
if (nextIdx != 0)
{
Metadata* next = &_spareMemory[-nextIdx];
next->PreviousLevel = nodeIdx;
}
node->NextLevel = nextIdx;
_spareMemory->NextLevel = nodeIdx;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void RemoveFromBucket(
Metadata* node,
int nodeIdx
)
{
RemoveFromBucket(GetLevel(node->Size), node, nodeIdx);
}
private void RemoveFromBucket(
int level,
Metadata* node,
int nodeIdx
)
{
int currentLevel = level;
int currentNextLevelIdx = node->NextLevel;
int currentPrevLevelIdx = node->PreviousLevel;
if (_bucket[currentLevel] == nodeIdx)
{
_bucket[currentLevel] = currentNextLevelIdx;
} else if (currentPrevLevelIdx != 0)
{
Metadata* prevLevel = &_spareMemory[-currentPrevLevelIdx];
prevLevel->NextLevel = currentNextLevelIdx;
}
if (currentNextLevelIdx != 0)
{
Metadata* nextLevel = &_spareMemory[-currentNextLevelIdx];
nextLevel->PreviousLevel = currentPrevLevelIdx;
}
node->PreviousLevel = 0;
node->NextLevel = 0;
}
// Assuming node to remove is head f bucket
private void RemoveFromBucketHead(
int level,
Metadata* node,
int nodeIdx
)
{
int currentLevel = level;
int currentNextLevelIdx = node->NextLevel;
// int currentPrevLevelIdx = 0;
if (_bucket[currentLevel] == nodeIdx)
{
_bucket[currentLevel] = currentNextLevelIdx;
}
/*
else if (currentPrevLevelIdx != 0)
{
Metadata* prevLevel = &_spareMemory[-currentPrevLevelIdx];
prevLevel->NextLevel = currentNextLevelIdx;
}
/**/
if (currentNextLevelIdx != 0)
{
Metadata* nextLevel = &_spareMemory[-currentNextLevelIdx];
nextLevel->PreviousLevel = 0;
}
node->PreviousLevel = 0;
node->NextLevel = 0;
}
private bool TryAllocNewChunk(
ulong size,
out Metadata* meta
)
{
int idx = _spareMemory->NextLevel;
ulong spareSize = _spareMemory->Size;
if (idx != 0)
{
if (spareSize < size)
{
meta = null;
return false;
}
meta = &_spareMemory[-idx];
int nextSpareIdx = meta->NextLevel;
if (nextSpareIdx != 0)
{
Metadata* nextSpare = &_spareMemory[-nextSpareIdx];
nextSpare->PreviousLevel = 0;
}
_spareMemory->NextLevel = nextSpareIdx;
} else
{
if (spareSize < Metadata.SizeOf + size)
{
meta = null;
return false;
}
_metadataEnd++;
idx = _metadataEnd;
meta = &_spareMemory[-idx];
spareSize -= Metadata.SizeOf;
}
spareSize -= size;
meta->Size = size;
meta->Position = _spareMemory->Position;
meta->State = AllocationState.Free;
meta->PreviousLevel = 0;
meta->NextLevel = 0;
_spareMemory->Size = spareSize;
_spareMemory->Position += size;
// insert the new chunk to position chain
int nearIdx = _spareMemory->PreviousNear;
Metadata* near = &_spareMemory[-nearIdx];
near->NextNear = idx;
meta->PreviousNear = nearIdx;
_spareMemory->PreviousNear = idx;
meta->NextNear = 0;
return true;
}
private bool TryGetNewMetadata(
out Metadata* meta
)
{
ulong spareSize = _spareMemory->Size;
if (spareSize < Metadata.SizeOf)
{
meta = null;
return false;
}
_metadataEnd++;
meta = &_spareMemory[-_metadataEnd];
spareSize -= Metadata.SizeOf;
_spareMemory->Size = spareSize;
return true;
}
private bool TryTakeBucketEntry(
ulong minimumSize,
out Metadata* entry
)
{
int level = GetLevel(minimumSize);
// first try to find in the target level
{
int idx = _bucket[level];
while (idx != 0)
{
Metadata* candidate = &_spareMemory[-idx];
if (candidate->Size >= minimumSize)
{
RemoveFromBucket(level, candidate, idx);
entry = candidate;
return true;
}
idx = candidate->NextLevel;
}
}
for (int i = level + 1; i < _bucket.Length; i++)
{
int idx = _bucket[i];
if (idx != 0)
{
Metadata* candidate = &_spareMemory[-idx];
RemoveFromBucketHead(i, candidate, idx);
entry = candidate;
return true;
}
}
entry = null;
return false;
}
[StructLayout(LayoutKind.Explicit)]
public readonly struct Handle
{
[FieldOffset(0)] internal readonly Metadata* _meta;
[FieldOffset(8)] public readonly Ptr64 Address;
[FieldOffset(16)] public readonly Ptr64 Size;
internal Handle(
Metadata* meta,
Ptr64 baseAddress
)
{
_meta = meta;
Address = (ulong)meta->Position;
Size = meta->Size;
}
[Obsolete("You should not create instance from default. Alloc from allocator please")]
public Handle() { }
public ref byte this[
int offset
] => ref ((byte*)(ulong)Address)[offset];
public Span<byte> AsSpan(
int range = -1
)
{
return new Span<byte>((byte*)(ulong)Address, range < 0 ? (int)(ulong)Size : range);
}
}
internal enum AllocationState : byte
{
SpareOrCollect = 0,
Free = 1,
Alloc = 2,
}
[StructLayout(LayoutKind.Explicit, Size = 32)]
internal struct Metadata
{
[FieldOffset(16)] private ulong PositionMask;
[FieldOffset(24)] private ulong SizeMask;
[FieldOffset(30)] internal AllocationState State;
[FieldOffset(31)] internal sbyte Level;
[FieldOffset(0)] public int NextLevel;
[FieldOffset(4)] public int NextNear;
[FieldOffset(8)] public int PreviousLevel;
[FieldOffset(12)] public int PreviousNear;
public Ptr64 Position
{
readonly get => PositionMask;
set => PositionMask = value;
}
public Ptr64 Size
{
readonly get => (Ptr64)(SizeMask & Mask48Bits);
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
}
public Ptr64 SizeForSpare
{
readonly get => SizeMask;
set => SizeMask = value;
}
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
}
#region alloc and free
public Handle Alloc(
nuint size
)
{
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
if (allocSize > _maxAllocSize) {
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
}
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
!TryAllocNewChunk(allocSize, out entry)) {
throw new OverflowException("No available ");
}
int entryIdx = (int)(_spareMemory - entry);
ulong entrySize = entry->Size;
if (entrySize < allocSize) {
throw new OverflowException("Bucket returned an undersized block.");
}
ulong remain = entrySize - allocSize;
// slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
{
int newEntryIdx = (int)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize;
newEntry->Size = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
bucketHead->PreviousLevel = newEntryIdx;
}
newEntry->NextLevel = bucketHeadIdx;
newEntry->PreviousLevel = 0;
_bucket[newLevel] = newEntryIdx;
// push to position chain
int afterIdx = entry->NextNear;
Metadata* after = &_spareMemory[-afterIdx];
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize) {
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
return new Handle(entry, _basePtr);
}
public void Collect()
{
int currentIdx = _spareMemory->NextNear;
do
{
if (currentIdx == 0)
{
break;
}
Metadata* current = &_spareMemory[-currentIdx];
if (current->State == AllocationState.Free)
{
// scan next
Ptr64 size = 0;
int nextIdx = current->NextNear;
while (nextIdx != 0)
{
Metadata* next = &_spareMemory[-nextIdx];
if (next->State != AllocationState.Free)
{
break;
}
size += next->Size;
RecycleNode(next);
nextIdx = current->NextNear;
}
if (size > 0)
{
current->Size += size;
RemoveFromBucket(current, currentIdx);
int newLevel = GetLevel(current->Size);
int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
bucketHead->PreviousLevel = currentIdx;
}
current->NextLevel = bucketHeadIdx;
current->PreviousLevel = 0;
_bucket[newLevel] = currentIdx;
}
}
currentIdx = current->NextNear;
} while (true);
}
public void Free(
Handle handle,
int collectScanWindow = 4
)
{
Metadata* head = handle._meta;
int headIdx = (int)(_spareMemory - head);
Ptr64 size = handle.Size;
for (int i = 0; i < collectScanWindow; i++)
{
// forward scan, previous direction
int forwardIdx = head->PreviousNear;
if (forwardIdx == 0)
{
break;
}
Metadata* prev = &_spareMemory[-forwardIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->Size;
head->Position = prev->Position;
RecycleNode(prev);
}
for (int i = 0; i < collectScanWindow; i++)
{
// backward scan, next direction
int nextIdx = head->NextNear;
if (nextIdx == 0)
{
break;
}
Metadata* prev = &_spareMemory[-nextIdx];
if (prev->State != AllocationState.Free)
{
break;
}
size += prev->Size;
RecycleNode(prev);
}
// recycle to level;
head->Size = size;
head->State = AllocationState.Free;
head->PreviousLevel = 0;
int level = GetLevel(head->Size);
int bucketHeadIdx = _bucket[level];
head->NextLevel = 0;
if (bucketHeadIdx != 0)
{
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
bucketHead->PreviousLevel = headIdx;
head->NextLevel = bucketHeadIdx;
}
_bucket[level] = headIdx;
return;
}
public bool TryAlloc(
nuint size,
out Handle h
)
{
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
if (allocSize > _maxAllocSize)
{
h = default;
return false;
}
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
!TryAllocNewChunk(allocSize, out entry))
{
h = default;
return false;
}
int entryIdx = (int)(_spareMemory - entry);
ulong entrySize = entry->Size;
if (entrySize < allocSize)
{
h = default;
return false;
}
ulong remain = entrySize - allocSize;
// slice tail
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
{
int newEntryIdx = (int)(_spareMemory - newEntry);
newEntry->Position = entry->Position + allocSize;
newEntry->Size = remain;
newEntry->State = AllocationState.Free;
int newLevel = GetLevel(remain);
int bucketHeadIdx = _bucket[newLevel];
if (bucketHeadIdx != 0)
{
// if there is already an entry in the bucket, link it to the new entry
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
newEntry->NextLevel = bucketHeadIdx;
bucketHead->PreviousLevel = newEntryIdx;
} else
{
newEntry->NextLevel = 0;
}
newEntry->PreviousLevel = 0;
_bucket[newLevel] = newEntryIdx;
// push to position chain
int afterIdx = entry->NextNear;
Metadata* after = &_spareMemory[-afterIdx];
newEntry->NextNear = afterIdx;
after->PreviousNear = newEntryIdx;
entry->NextNear = newEntryIdx;
newEntry->PreviousNear = entryIdx;
}
entry->State = AllocationState.Alloc;
if (entry->Position - (ulong)_basePtr > _totalSize)
{
h = default;
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
}
h = new Handle(entry, _basePtr);
return true;
}
#endregion
}
}
@@ -0,0 +1,82 @@
// -----------------------------------------------------------------------------
// file="IRegisterType"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
/// <summary>
/// Abstraction for a wide SIMD register or equivalent packed numeric storage. Provides raw
/// reference access, broadcast initialization and element level access. Typical
/// implementations map to 128/256/512-bit hardware registers or software emulated blocks.
/// </summary>
public interface IRegisterPresentation
{
/// <summary>
/// Gets a reference to the first element of the register reinterpreted as type <typeparamref
/// name="T" />.
/// </summary>
/// <typeparam name="T">
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
/// </typeparam>
/// <returns>
/// Writable reference to element 0 (lowest lane / lowest bits).
/// </returns>
/// <remarks>
/// Intended for treating the register as a contiguous block of elements. The returned reference
/// must remain valid for the lifetime of the register and be identical to <see
/// cref="Member{T}(int)" /> with index 0. Caller must choose a type layout compatible with the
/// actual register representation.
/// </remarks>
ref T AsRef<T>() where T : unmanaged, INumber<T>;
/// <summary>
/// Fills all elements (lanes) of the register with the provided scalar <paramref name="value" />.
/// </summary>
/// <typeparam name="T">
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
/// </typeparam>
/// <param name="value">
/// Scalar value to replicate to every element.
/// </param>
/// <remarks>
/// Should map to a single broadcast/shuffle instruction where possible. After completion all
/// lanes read back the same value.
/// </remarks>
void BroadCastFrom<T>(
T value
) where T : unmanaged, INumber<T>;
/// <summary>
/// Gets a writable reference to the element at the specified <paramref name="index" />.
/// </summary>
/// <typeparam name="T">
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
/// </typeparam>
/// <param name="index">
/// Zero-based lane index (range defined by implementation).
/// </param>
/// <returns>
/// Writable reference to the lane.
/// </returns>
/// <remarks>
/// Enables scalar lane edits without unpacking the entire register. Implementations may omit
/// bounds checks for performance. Caller is responsible for passing a valid index.
/// </remarks>
ref T Member<T>(
int index
) where T : unmanaged, INumber<T>;
}
}
@@ -0,0 +1,44 @@
// -----------------------------------------------------------------------------
// file="LogicalCore"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public readonly struct LogicalCore
{
public readonly uint GlobalCpuId;
public readonly uint NumaGroupId;
/// <summary>
/// Stays 0 on linux.
/// </summary>
public readonly uint NumaNodeId;
internal LogicalCore(
uint globalCpuId,
uint numaGroupId,
uint numaNodeId
)
{
GlobalCpuId = globalCpuId;
NumaGroupId = numaGroupId;
NumaNodeId = numaNodeId;
}
[Obsolete(
"Do not manually create instances of LogicalCore. Use NumaNode to obtain LogicalCore instances.",
true)]
public LogicalCore() { }
}
}
@@ -0,0 +1,118 @@
// -----------------------------------------------------------------------------
// file="M128"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Runtime.Intrinsics.X86;
using System.Text;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
[StructLayout(LayoutKind.Explicit, Pack = 16)]
public unsafe struct M128 : IRegisterPresentation
{
[FieldOffset(0)] internal fixed byte Buffer[16];
[FieldOffset(0)] internal Vector128<float> Vec;
internal M128(
Vector4 v
)
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out Vec);
Vec = v.AsVector128();
}
internal M128(
Vector128<float> v
)
{
Unsafe.SkipInit(out this);
Vec = v;
}
public ref T AsRef<T>() where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref Buffer[0]);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public readonly Vector4 AsVector4()
{
return Vec.AsVector4();
}
public void BroadCastFrom<T>(
T value
) where T : unmanaged, INumber<T>
{
Vec = Vector128.Create<T>(value).AsSingle();
}
public override readonly bool Equals(
[NotNullWhen(true)] object obj
)
{
return obj is M128 other && this == other;
}
public override readonly int GetHashCode()
{
return Vec.GetHashCode();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T Member<T>(
int index
) where T : unmanaged, INumber<T>
{
return ref Unsafe.Add(ref AsRef<T>(), index);
}
public override string ToString()
{
return string.Empty;
}
public string ToString(
int sample
)
{
string str = string.Empty;
for (int i = 15; i >= 0; i--) {
str += $"{Convert.ToString(Buffer[i], sample)} ";
}
return str;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool operator !=(
M128 left,
M128 right
)
{
return left.Vec != right.Vec;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static bool operator ==(
M128 left,
M128 right
)
{
return left.Vec == right.Vec;
}
}
}
@@ -0,0 +1,49 @@
// -----------------------------------------------------------------------------
// file="M256"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
[StructLayout(LayoutKind.Explicit, Pack = 16)]
public unsafe struct M256 : IRegisterPresentation
{
[FieldOffset(0)] internal fixed byte Buffer[32];
[FieldOffset(0)] internal Vector256<float> Vec;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T AsRef<T>() where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref Buffer[0]);
}
public void BroadCastFrom<T>(
T value
) where T : unmanaged, INumber<T>
{
Vec = Vector256.Create(value).AsSingle();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T Member<T>(
int index
) where T : unmanaged, INumber<T>
{
return ref Unsafe.Add(ref AsRef<T>(), index);
}
}
}
@@ -0,0 +1,53 @@
// -----------------------------------------------------------------------------
// file="M512"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)]
public unsafe struct M512 : IRegisterPresentation
{
[FieldOffset(0)] private fixed byte _buffer[64];
[FieldOffset(0)] private Vector512<byte> Vec;
public M512()
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out Vec);
}
public ref T AsRef<T>() where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref _buffer[0]);
}
public void BroadCastFrom<T>(
T value
) where T : unmanaged, INumber<T>
{
Vec = Vector512.Create(value).AsByte();
}
public ref T Member<T>(
int index
) where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref _buffer[index]);
}
}
}
@@ -0,0 +1,114 @@
// -----------------------------------------------------------------------------
// file="M64"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
[StructLayout(LayoutKind.Explicit, Pack = 8)]
public unsafe struct M64 : IRegisterPresentation
{
[FieldOffset(0)] private fixed byte buffer[8];
[FieldOffset(0)] private ulong value;
[FieldOffset(0)] private Vector64<float> vec;
public M64()
{
Unsafe.SkipInit(out this);
Unsafe.SkipInit(out vec);
Unsafe.SkipInit(out value);
}
public ref T AsRef<T>() where T : unmanaged,INumber<T>
{
return ref Unsafe.As<byte, T>(ref buffer[0]);
}
public void BroadCastFrom<T>(
T value
) where T : unmanaged, INumber<T>
{
vec = Vector64.Create(value).AsSingle();
}
public override bool Equals(
object obj
)
{
return obj is M64 m64 && this == m64;
}
public override readonly int GetHashCode()
{
return value.GetHashCode();
}
public ulong GetLowerBits(
int bitCount
)
{
return value & ((1UL << bitCount) - 1);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public ref T Member<T>(
int index
) where T : unmanaged, INumber<T>
{
return ref Unsafe.As<byte, T>(ref buffer[index * sizeof(T)]);
}
public void SetLowerBits(
int bitCount,
ulong bits
)
{
value = (value & (~(1UL << bitCount))) | (bits & (1UL << bitCount));
}
public static bool operator !=(
M64 left,
M64 right
)
{
return !(left == right);
}
public static M64 operator <<(
M64 value,
int shift
)
{
M64 result = new M64();
result.value = value.value << shift;
return result;
}
public static bool operator ==(
M64 left,
M64 right
)
{
return left.value == right.value;
}
public static M64 operator >>(
M64 value,
int shift
)
{
return new()
{
value = value.value << shift
};
}
}
}
@@ -0,0 +1,158 @@
// -----------------------------------------------------------------------------
// file="Numa"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public static partial class Numa
{
private static NumaNode[] _numaNodes;
public static int NumaNodeCount { get; private set; }
public static ReadOnlySpan<NumaNode> AllNumaNode => _numaNodes;
public static unsafe void* Alloc(
int numaId,
nuint size
)
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
return NumaAllocNative(size, numaId);
}
public static T[][] CreateForeachThread<T>(
Func<LogicalCore, T> factory
)
{
if (factory is null) {
return [];
}
T[][] arr = new T[_numaNodes.Length][];
for (int i = 0; i < _numaNodes.Length; i++)
{
NumaNode node = _numaNodes[i];
T[] values = new T[node.LogicalCoreCount];
ReadOnlySpan<LogicalCore> cores = node.LogicalCores;
for (int j = 0; j < cores.Length; j++) {
values[j] = factory(cores[j]);
}
arr[i] = values;
}
return arr;
}
public static unsafe void Free(
void* address,
int numaId,
nuint size
)
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
}
NumaFreeNative(address, size, numaId);
}
[LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_each")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
public static partial int GetNumaEach(
uint numaId,
Span<ulong> CpuIndexInNuma,
Span<uint> globalCpuIdInNuma
);
[LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_overall")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
public static partial int GetNumaOverall(
ref int count,
Span<int> coreCountInNuma
);
public static L4[] RequestBufferForEachNuma()
{
if (_numaNodes.Length == 0) {
throw new InvalidOperationException("Numa info has not been loaded.");
}
L4[] buffers = new L4[NumaNodeCount];
for (int i = 0; i < NumaNodeCount; i++)
{
L4 buffer = new L4(i);
buffers[i] = buffer;
}
return buffers;
}
internal static void Initialize()
{
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
int numaNodeCount = 0;
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
throw new InvalidOperationException("Failed to get NUMA overall information.");
}
_numaNodes = new NumaNode[numaNodeCount];
for (uint i = 0; i < numaNodeCount; i++) {
_numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]);
}
}
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial void SetThreadAffinity(
int coreId
);
[MethodImpl(MethodImplOptions.NoInlining)]
private static NumaNode InitNumaNode(
uint numaId,
int count
)
{
Span<uint> globalIndex = stackalloc uint[count];
_ = GetNumaEach(numaId, null, globalIndex);
return new(numaId, count, globalIndex);
}
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_alloc")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static unsafe partial void* NumaAllocNative(
nuint size,
int numaId
);
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_free")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static unsafe partial void NumaFreeNative(
void* ptr,
nuint size,
int numa_id
);
}
}
@@ -0,0 +1,40 @@
// -----------------------------------------------------------------------------
// file="NumaNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.HighPerformance.ProcessorAbstraction
{
public class NumaNode
{
private LogicalCore[] _logicalCores;
internal NumaNode(
uint numaId,
int count,
Span<uint> logicalCoreIds
)
{
_logicalCores = new LogicalCore[count];
for (int i = 0; i < count; i++) {
_logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]);
}
}
public int LogicalCoreCount => _logicalCores.Length;
public ReadOnlySpan<LogicalCore> LogicalCores => _logicalCores;
}
}
+113
View File
@@ -0,0 +1,113 @@
// -----------------------------------------------------------------------------
// file="DefaultLog"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Internal
{
internal static unsafe partial class DefaultLog
{
private const long MaxLogFileSize = 10 * 1024 * 1024; // 10 MB
private static LogType _currentLogLevel = LogType.Info;
internal static readonly GCHandle ssc_logHandle =
GCHandle.Alloc(new byte[512], GCHandleType.Pinned);
internal static SemaphoreSlim logSyncSemaphore = new(1, 1);
internal static void InternalAppendLog(
string log
)
{
if (!File.Exists(Native.LogPath))
{
FileStream stream = File.Create(Native.LogPath);
stream.Close();
}
using (StreamWriter logStream = File.AppendText(Native.LogPath)) {
logStream.WriteLine(log);
}
}
internal static void InternalWriteLog(
string logLine,
LogType logType
)
{
if (logType < _currentLogLevel) {
return;
}
logSyncSemaphore.Wait();
try
{
RotateLogFile();
if (!File.Exists(Native.LogPath))
{
FileStream stream = File.Create(Native.LogPath);
stream.Close();
}
using (StreamWriter logStream = File.AppendText(Native.LogPath)) {
logStream.WriteLine($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{logType}] {logLine}");
}
}
finally
{
_ = logSyncSemaphore.Release();
}
}
internal static void InternalWriteLog(
string beforeTime,
string logLine,
LogType logType
)
{
logSyncSemaphore.Wait();
if (!File.Exists(Native.LogPath))
{
FileStream stream = File.Create(Native.LogPath);
stream.Close();
}
using (StreamWriter logStream = File.AppendText(Native.LogPath))
{
logStream.WriteLine(beforeTime);
logStream.WriteLine($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{logType}] {logLine}");
}
_ = logSyncSemaphore.Release(1);
}
internal static void SetLogLevel(
LogType logLevel
)
{
_currentLogLevel = logLevel;
}
private static void RotateLogFile()
{
if (File.Exists(Native.LogPath) && new FileInfo(Native.LogPath).Length > MaxLogFileSize)
{
string backupFile = Native.LogPath + ".backup";
if (File.Exists(backupFile)) {
File.Delete(backupFile);
}
File.Move(Native.LogPath, backupFile);
}
}
}
}
+28
View File
@@ -0,0 +1,28 @@
// -----------------------------------------------------------------------------
// file="Native"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Internal
{
internal static unsafe partial class Native
{
internal const string LibName =
"StgSharp.Native";
internal const string LogPath =
"SS_error.log";
internal static IntPtr _libPtr = IntPtr.Zero;
}
}
+26
View File
@@ -0,0 +1,26 @@
// -----------------------------------------------------------------------------
// file="public"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp
{
public enum LogType
{
Error,
Warning,
Info,
InfoError,
InfoWarning,
}
}
@@ -0,0 +1,72 @@
// -----------------------------------------------------------------------------
// file="CrossFramePipelineNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.PipeLine;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public class CrossFrameOperation : IConvertableToPipelineNode
{
private static string[] _inputName = ["TaskIn"];
private static string[] _outputName = ["TaskOut"];
private Action _mainExecution;
private int currentCount;
private readonly int cycleCount;
public CrossFrameOperation(
Action startup,
int cycleCount
)
{
MainExecution = startup;
this.cycleCount = (cycleCount > 1) ?
cycleCount :
throw new ArgumentException(
$"{$"The {typeof(CrossFrameOperation).Name} crosses only one cycle, "}{$"please use a {typeof(PipelineNode).Name} instead."}");
this.cycleCount = cycleCount;
currentCount = cycleCount;
}
public Action MainExecution
{
get => _mainExecution;
private protected set => _mainExecution = value;
}
public IEnumerable<string> InputPortName => _inputName;
public IEnumerable<string> OutputPortName => _outputName;
public PipelineNodeOperation Operation => NodeMain;
public void NodeMain(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
)
{
if (currentCount < cycleCount)
{
Interlocked.Increment(ref currentCount);
} else
{
Interlocked.Exchange(ref currentCount, 0);
Task.Run(MainExecution);
}
// Console.WriteLine(currentCount);
PipelineNodeOutPort.SkipAll(output);
}
}
}
@@ -0,0 +1,69 @@
// -----------------------------------------------------------------------------
// file="IConvertableToPipelineNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Text;
namespace StgSharp.PipeLine
{
public delegate void PipelineNodeOperation(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
);
public interface IConvertableToPipelineNode
{
public IEnumerable<string> InputPortName { get; }
public IEnumerable<string> OutputPortName { get; }
PipelineNodeOperation Operation { get; }
void NodeMain(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
);
}
internal class DefaultConvertableToBlueprintNode : IConvertableToPipelineNode
{
private string[] _input;
private string[] _output;
private PipelineNodeOperation _execution;
public DefaultConvertableToBlueprintNode(
PipelineNodeOperation execution,
string[] input,
string[] output
)
{
_input = input;
_output = output;
_execution = execution;
}
public IEnumerable<string> InputPortName => _input;
public IEnumerable<string> OutputPortName => _output;
public PipelineNodeOperation Operation => _execution;
public void NodeMain(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
)
{
_execution.Invoke(input, output);
}
}
}
@@ -0,0 +1,231 @@
// -----------------------------------------------------------------------------
// file="PipeLineRunner"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Threading;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public static class PipeLineRunner
{
private static NodeRunner[] allAvailableTask = CreateBlueprintRunnerArray();
private static CancellationTokenSource cts = new CancellationTokenSource();
private static ConcurrentStack<PipelineScheduler> pipelineInWaiting
= new ConcurrentStack<PipelineScheduler>();
private static int _callingThread;
private static SemaphoreSlim pipelineLock = new SemaphoreSlim(0, 1);
public static void Run(
PipelineScheduler pipeline
)
{
// Console.WriteLine();
if (pipeline == null)
{
return;
}
if (pipeline.IsRunning) {
throw new InvalidOperationException("Attempt to run a blueprint instance twice.");
}
if (!IsValidCalling()) {
pipelineLock.Wait();
}
if (InterruptAndRunNewPipeline(pipeline))
{
while (InterruptAndRunNewPipeline(NodeRunner.CurrentPipeline!)) { }
}
pipelineLock.Release();
Interlocked.Exchange(ref _callingThread, -1);
}
public static void SetMaxTaskCount(
int size
)
{
if (size < 1) {
size = 1;
}
if (allAvailableTask.Length >= size)
{
for (int i = size; i < allAvailableTask.Length; i++)
{
if (allAvailableTask[i] != null) {
allAvailableTask[i].Cancel();
}
}
}
Array.Resize(ref allAvailableTask, size);
}
public static void Terminate()
{
cts.Cancel();
if (NodeRunner.CurrentPipeline is null) {
return;
}
NodeRunner.CurrentPipeline.Terminate();
foreach (NodeRunner task in allAvailableTask) {
task.Cts.Cancel();
}
foreach (PipelineScheduler bp in pipelineInWaiting) {
bp.Terminate();
}
pipelineInWaiting.Clear();
}
internal static bool InterruptAndRunNewPipeline(
PipelineScheduler newPipeline
)
{
ArgumentNullException.ThrowIfNull(newPipeline);
if ((newPipeline != NodeRunner.CurrentPipeline) && (NodeRunner.CurrentPipeline != null))
{
// if a running bp exists
pipelineInWaiting.Push(NodeRunner.CurrentPipeline);
}
// load new bp to current bp
NodeRunner.CurrentPipeline = newPipeline;
Interlocked.Exchange(ref _callingThread, Environment.CurrentManagedThreadId);
// run beginning node to init it first.
if (!NodeRunner.CurrentPipeline.IsRunning)
{
(NodeRunner.CurrentPipeline.BeginLayer as BeginningNode)!.Run();
}
// refresh all tasks
/*
for( int i = 0; i < allAvailableTask.Length; i++ )
{
NodeRunner item = allAvailableTask[ i ];
if( ( item is null ) || item.IsCompleted ) {
allAvailableTask[ i ] = NodeRunner.Run();
}
}
/**/
PipelineNode node;
while (!cts.Token.IsCancellationRequested &&
NodeRunner.CurrentPipeline.RequestNextMainThreadNode(out node))
{
node.Run();
if (!NodeRunner.CurrentPipeline.IsRunning)
{
break;
}
}
while ((NodeRunner.CurrentPipeline != null) &&
NodeRunner.CurrentPipeline.IsRunning &&
!cts.Token.IsCancellationRequested &&
NodeRunner.CurrentPipeline!.RequestNexConcurrentNode(out node))
{
node.Run();
if (!NodeRunner.CurrentPipeline!.IsRunning)
{
break;
}
}
(NodeRunner.CurrentPipeline!.EndLayer as EndingNode)!.Run();
bool isEmpty = pipelineInWaiting.TryPop(out PipelineScheduler? pipeline);
NodeRunner.CurrentPipeline = pipeline!;
return isEmpty;
}
/// <summary>
/// <para> Only the following occasions of calling are valid: </para> <para>1. no bp is
/// running or the only bp is quitting.</para><para>2. calling from the main thread or a
/// thread the current bp is running on. </para>
/// </summary>
internal static bool IsValidCalling()
{
if ((NodeRunner.CurrentPipeline == null) || //no bp running at all
((pipelineInWaiting.IsEmpty) && (NodeRunner.CurrentPipeline.IsRunning == false)))
{
// a bp is quitting
// in case no bp is running
return true;
}
int? id = Task.CurrentId;
if (id is null)
{
// in case calling from main thread and current running bp
return _callingThread == Environment.CurrentManagedThreadId;
}
foreach (NodeRunner item in allAvailableTask)
{
if (item.Id == id)
{
// if calling from any bp running task
return true;
}
}
return false;
}
private static NodeRunner[] CreateBlueprintRunnerArray()
{
int count = Environment.ProcessorCount;
return new NodeRunner[ count < 2 ? count : count - 2];
}
}
internal class NodeRunner : CancellableTask
{
private static volatile PipelineScheduler? _current;
internal NodeRunner()
: base()
{
_startup = InternalRun;
}
internal static PipelineScheduler? CurrentPipeline
{
get => _current;
set => _current = value;
}
internal static NodeRunner Run()
{
NodeRunner ret = new NodeRunner
{
_startup = InternalRun
};
ret._internalTask = Task.Run(ret.InternalAction);
return ret;
}
private static void InternalRun(
CancellationToken token
)
{
PipelineNode item;
Console.WriteLine($"Global thread {Environment.CurrentManagedThreadId} starts.");
while ((CurrentPipeline != null) &&
CurrentPipeline.IsRunning &&
!token.IsCancellationRequested &&
CurrentPipeline.RequestNexConcurrentNode(out item)) {
item.Run();
}
}
}//------------------------------------- End of Class ------------------------------------------
}
@@ -0,0 +1,107 @@
// -----------------------------------------------------------------------------
// file="PipelineMarshal"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public static class PipelineMarshal
{
public static IEnumerable<PipelineNode> GetNodeSortedEnumeration(
PipelineScheduler scheduler
)
{
ArgumentNullException.ThrowIfNull(scheduler);
return new PipelineNodeEnumerator(scheduler)
{
_count = scheduler._allNode.Count,
};
}
private class PipelineNodeEnumerator : ICollection<PipelineNode>
{
private readonly List<List<PipelineNode>> _mt, _c;
public int _count;
public PipelineNodeEnumerator(
PipelineScheduler scheduler
)
{
if (!scheduler.TryGetAllNode(out _mt, out _c)) {
throw new InvalidOperationException("Binded scheduler is not sorted.");
}
}
public bool IsReadOnly => true;
public int Count => _count;
public IEnumerator<PipelineNode> GetEnumerator()
{
for (int i = 1; i < _mt.Count; i++)
{
foreach (PipelineNode node in _mt[i]) {
yield return node;
}
foreach (PipelineNode node in _c[i]) {
yield return node;
}
}
}
void ICollection<PipelineNode>.Add(
PipelineNode item
)
{
throw new NotSupportedException();
}
void ICollection<PipelineNode>.Clear()
{
throw new NotSupportedException();
}
bool ICollection<PipelineNode>.Contains(
PipelineNode item
)
{
throw new NotSupportedException();
}
void ICollection<PipelineNode>.CopyTo(
PipelineNode[] array,
int arrayIndex
)
{
throw new NotSupportedException();
}
bool ICollection<PipelineNode>.Remove(
PipelineNode item
)
{
throw new NotSupportedException();
}
IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
}
}
@@ -0,0 +1,276 @@
// -----------------------------------------------------------------------------
// file="PipelineNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
/// <summary>
///
/// </summary>
/// <risk>
/// Reset pipeline has risk, this operation is a direct operation to port in current node, the
/// port on the other node cannot get the change.
/// </risk>
#pragma warning disable CA1036
public class PipelineNode
#pragma warning restore CA1036
{
private static readonly Dictionary<string, PipelineConnector> _emptyLayer = [];
private static PipelineNode _emptyNode = new PipelineNode(
PipelineNodeLabel.Empty, null!, false);
private IConvertableToPipelineNode _mainOperation;
private PipelineNodeLabel _label;
private protected bool _nativeRequested;
private protected Dictionary<string, PipelineNodeInPort> _input;
private protected Dictionary<string, PipelineNodeOutPort> _output;
private protected int _level, _count;
internal PipelineNode(
PipelineNodeLabel label,
IConvertableToPipelineNode node,
bool isNative
)
{
_label = label;
_nativeRequested = isNative;
_mainOperation = node;
InputPorts = new();
OutputPorts = new();
foreach (string port in node.InputPortName) {
InputPorts.Add(port, new PipelineNodeInPort(this, port));
}
foreach (string port in node.OutputPortName) {
OutputPorts.Add(port, new PipelineNodeOutPort(this, port));
}
}
#pragma warning disable CS8618
internal PipelineNode(
PipelineNodeOperation operation,
PipelineNodeLabel name,
bool isNative,
string[] inputNames,
string[] outputNames
)
#pragma warning restore CS8618
{
if ((inputNames == null) || (inputNames.Length == 0)) {
throw new ArgumentNullException(nameof(inputNames));
}
if ((outputNames == null) || (outputNames.Length == 0)) {
throw new ArgumentNullException(nameof(outputNames));
}
_label = name;
_nativeRequested = isNative;
if (operation == null) {
operation = DefaultOperation;
}
_mainOperation = new DefaultConvertableToBlueprintNode(
operation, inputNames, outputNames);
InputPorts = new Dictionary<string, PipelineNodeInPort>();
OutputPorts = new Dictionary<string, PipelineNodeOutPort>();
foreach (string port in inputNames) {
InputPorts.Add(port, new PipelineNodeInPort(this, port));
}
foreach (string port in outputNames) {
OutputPorts.Add(port, new PipelineNodeOutPort(this, port));
}
}
public bool IsNative
{
get => _nativeRequested;
internal set => _nativeRequested = value;
}
public static Dictionary<string, PipelineConnector> EmptyPortLayer => _emptyLayer;
public IConvertableToPipelineNode NodeOperation => _mainOperation;
public int Level
{
get => _level;
private set => _level = value;
}
public static PipelineNode Empty => _emptyNode;
public PipelineNodeLabel Label => _label;
internal Dictionary<string, PipelineNodeInPort> InputPorts
{
get => _input;
private set => _input = value;
}
internal Dictionary<string, PipelineNodeOutPort> OutputPorts
{
get => _output;
private set => _output = value;
}
internal HashSet<PipelineNode> Previous { get; } = [];
internal HashSet<PipelineNode> Next { get; } = [];
internal IConvertableToPipelineNode Value => _mainOperation;
internal int RemainingIncreaseCount => Previous.Count - _count;
public static void Connect(
PipelineNode front,
string frontNodePort,
string backNodePort,
PipelineNode back
)
{
ArgumentNullException.ThrowIfNull(front);
ArgumentNullException.ThrowIfNull(back);
if (!front.OutputPorts
.TryGetValue(frontNodePort,
out PipelineNodeOutPort? frontPort)) {
throw new ArgumentOutOfRangeException($"Cannot find port named {frontNodePort}");
}
if (!back.InputPorts.TryGetValue(backNodePort, out PipelineNodeInPort? backPort)) {
throw new ArgumentOutOfRangeException($"Cannot find port named {backNodePort}");
}
backPort.Connect(frontPort);
frontPort.Connect(backPort);
}
public IEnumerator<PipelineNode> GetFormerNodes()
{
foreach (PipelineNodeInPort port in InputPorts.Values)
{
IEnumerator<PipelineNode> enumerator = port.GetFormerNodes();
while (enumerator.MoveNext()) {
yield return enumerator.Current;
}
}
}
public virtual void Run()
{
try
{
// Console.WriteLine(_label);
PipelineNodeInPort.WaitAll(InputPorts);
if (_mainOperation != null) {
_mainOperation.NodeMain(in _input, in _output);
}
}
catch (Exception)
{
throw;
}
}
internal bool TryIncreaseLevelTo(
int level
)
{
_count++;
if (_count > Previous.Count) {
return false;
}
Level = int.Max(level, this.Level);
return true;
}
protected static void DefaultOperation(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
)
{
PipelineNodeOutPort.SkipAll(output);
}
#region port operation
public PipelineNodeInPort GetInputPort(
string lable
)
{
if (!InputPorts.TryGetValue(lable, out PipelineNodeInPort? ret)) {
throw new ArgumentOutOfRangeException(
$"Cannot find the interface named {lable} in node {Label}.");
}
return ret;
}
public PipelineNodeOutPort GetOutputPort(
string lable
)
{
if (!OutputPorts.TryGetValue(lable, out PipelineNodeOutPort? ret)) {
throw new ArgumentOutOfRangeException(
$"Cannot find the interface named {lable} in node {Label}.");
}
return ret;
}
public PipelineNodeInPort? DefineCertainInputPort(
string label
)
{
if (InputPorts.ContainsKey(label)) {
return null;
}
PipelineNodeInPort port = new(this, label);
InputPorts[label] = port;
return port;
}
public PipelineNodeOutPort? DefineCertainOutputPort(
string label
)
{
if (OutputPorts.ContainsKey(label)) {
return null;
}
PipelineNodeOutPort port = new(this, label);
OutputPorts[label] = port;
return port;
}
#endregion
}//--------------------------------- end of class -----------------------------------
public class PipelineNodeRunException : Exception
{
public PipelineNodeRunException() { }
public PipelineNodeRunException(
Exception ex
)
: base(string.Empty, ex) { }
public PipelineNodeRunException(
string message
)
: base(message) { }
public PipelineNodeRunException(
string message,
Exception innerException
)
: base(message, innerException) { }
}
}
@@ -0,0 +1,62 @@
// -----------------------------------------------------------------------------
// file="PipelineNodeLabel"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public abstract record class PipelineNodeLabel
{
public static PipelineNodeLabel Empty => PipelineNodeLabelEmpty.Only;
public abstract int ComputeHashCode();
public override int GetHashCode()
{
return ComputeHashCode();
}
}
public record class PipelineNodeStringLabel : PipelineNodeLabel
{
public PipelineNodeStringLabel(
string name
)
{
Name = name;
}
public string Name { get; }
public override int ComputeHashCode()
{
return string.GetHashCode(Name);
}
}
internal record class PipelineNodeLabelEmpty : PipelineNodeLabel
{
public static PipelineNodeLabelEmpty Only { get; } = new PipelineNodeLabelEmpty();
public override int ComputeHashCode()
{
return 0;
}
}
}
@@ -0,0 +1,187 @@
// -----------------------------------------------------------------------------
// file="PipelineNodePort"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public class PipelineNodeInPort
{
private HashSet<PipelineNodeOutPort> _linkerSet = new HashSet<PipelineNodeOutPort>();
public PipelineNodeInPort(
PipelineNode container,
string label
)
{
Container = container;
Label = label;
}
public IEnumerable<PipelineNodeOutPort> LinkedPorts => _linkerSet;
public PipelineNode Container { get; init; }
public SemaphoreSlim Waiter { get; init; }
public string Label { get; init; }
internal HashSet<PipelineNodeOutPort> Linkers => _linkerSet;
public void Connect(
PipelineNodeOutPort port
)
{
ArgumentNullException.ThrowIfNull(port);
if (_linkerSet.Add(port))
{
port.Linkers.Add(this);
Container.Previous.Add(port.Container);
port.Container.Next.Add(Container);
}
}
public IEnumerator<PipelineNode> GetFormerNodes()
{
foreach (PipelineNodeOutPort port in _linkerSet) {
yield return port.Container;
}
}
public void Wait()
{
PipelineNodeOutPort.WaitAll(_linkerSet);
}
public static void WaitAll(
IDictionary<string, PipelineNodeInPort> ports
)
{
if (ports is null) {
return;
}
foreach (PipelineNodeInPort port in ports.Values) {
port.Wait();
}
}
public static void WaitAll(
IEnumerable<PipelineNodeInPort> ports
)
{
if (ports is null) {
return;
}
foreach (PipelineNodeInPort port in ports) {
port.Wait();
}
}
}
public class PipelineNodeOutPort
{
private HashSet<PipelineNodeInPort> _linkerSet = new HashSet<PipelineNodeInPort>();
private volatile int _passCount;
private SemaphoreSlim formerCompleteSemaphore;
public PipelineNodeOutPort(
PipelineNode container,
string label
)
{
Container = container;
Label = label;
formerCompleteSemaphore = new SemaphoreSlim(initialCount:0, maxCount:1);
}
public PipelineNode Container { get; init; }
public string Label { get; init; }
internal HashSet<PipelineNodeInPort> Linkers => _linkerSet;
private SemaphoreSlim Waiter => formerCompleteSemaphore;
public void Connect(
PipelineNodeInPort port
)
{
ArgumentNullException.ThrowIfNull(port);
if (_linkerSet.Add(port))
{
port.Linkers.Add(this);
Container.Next.Add(port.Container);
port.Container.Previous.Add(Container);
}
}
public void Release()
{
if (_passCount == 0)
{
Waiter.Release();
} else
{
return;
}
}
public static void SkipAll(
Dictionary<string, PipelineNodeOutPort> ports
)
{
if (ports is null) {
return;
}
foreach (PipelineNodeOutPort port in ports.Values) {
port.Release();
}
}
public void TransmitValue(
IPipeLineConnectionPayload value
)
{
foreach (PipelineNodeInPort item in _linkerSet) { }
}
public static void WaitAll(
IEnumerable<PipelineNodeOutPort> ports
)
{
if (ports is null) {
return;
}
foreach (PipelineNodeOutPort port in ports) {
port.Wait();
}
}
internal void Wait()
{
Waiter.Wait();
if (_passCount >= _linkerSet.Count)
{
Interlocked.Exchange(ref _passCount, 0);
} else
{
Interlocked.Increment(ref _passCount);
Waiter.Release();
}
}
}
}
@@ -0,0 +1,280 @@
// -----------------------------------------------------------------------------
// file="PipelineScheduler"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Xml;
using System.Xml.Linq;
namespace StgSharp.PipeLine
{
/// <summary>
/// A sets of complex logic designed for running in multi threads environment. Warning: do not
/// run a <see cref="PipelineScheduler" /> instance in multi threads more than once.
/// </summary>
public partial class PipelineScheduler : IConvertableToPipelineNode
{
private BeginningNode _beginning;
private bool _isAvailable = false;
private EndingNode _ending;
private int mtLevel, cLevel,mtIndex, cIndex;
private List<List<PipelineNode>> _concurrentNodes, _mainThreadNodes ;
private SemaphoreSlim runningStat,_concurrentLock = new SemaphoreSlim(1, 1);
internal Dictionary<PipelineNodeLabel, PipelineNode> _allNode;
public PipelineScheduler()
{
_allNode = new Dictionary<PipelineNodeLabel, PipelineNode>();
runningStat = new SemaphoreSlim(1, 1);
_beginning = new BeginningNode(this);
_ending = new EndingNode(this);
}
/**/
public PipelineNode this[
PipelineNodeLabel label
] => _allNode[label];
public bool IsRunning => runningStat.CurrentCount == 0;
public IEnumerable<string> InputPortName => _beginning.InputPorts.Keys;
public IEnumerable<string> OutputPortName => _beginning.OutputPorts.Keys;
public PipelineNode BeginLayer => _beginning;
public PipelineNode EndLayer => _ending;
public PipelineNodeOperation Operation => (this as IConvertableToPipelineNode).NodeMain;
/**/
internal SemaphoreSlim RunningStat => runningStat;
public void AddNode(
PipelineNode node,
bool isNative
)
{
ArgumentNullException.ThrowIfNull(node);
node.IsNative = isNative;
_allNode.TryAdd(node.Label, node);
}
public void ArrangeNodes()
{
if (_isAvailable) {
return;
}
int maxLevel = 1;
Queue<PipelineNode> completedNodes = new Queue<PipelineNode>();
foreach (PipelineNode item in _beginning.Next)
{
item.TryIncreaseLevelTo(1);
completedNodes.Enqueue(item);
}
// calc level of nodes O(V)
while (completedNodes.Count > 0)
{
PipelineNode u = completedNodes.Dequeue();
foreach (PipelineNode v in u.Next)
{
if (!v.TryIncreaseLevelTo(u.Level + 1)) {
throw new InvalidOperationException("Ring in DAG detected.");
}
if (v.RemainingIncreaseCount == 0)
{
completedNodes.Enqueue(v);
maxLevel = int.Max(maxLevel, v.Level);
}
}
}
// bucket sort of nodes, O(E)
_concurrentNodes = new List<List<PipelineNode>>(maxLevel + 1);
_mainThreadNodes = new List<List<PipelineNode>>(maxLevel + 1);
for (int i = 0; i <= maxLevel; i++)
{
_concurrentNodes.Add([]);
_mainThreadNodes.Add([]);
}
foreach (PipelineNode item in _allNode.Values)
{
if (item.IsNative)
{
_mainThreadNodes[item.Level].Add(item);
} else
{
_concurrentNodes[item.Level].Add(item);
}
}
foreach (List<PipelineNode> item in _concurrentNodes) {
item.TrimExcess();
}
foreach (List<PipelineNode> item in _mainThreadNodes) {
item.TrimExcess();
}
_isAvailable = true;
}
public PipelineNode Create(
PipelineNodeLabel label,
[NotNull] IConvertableToPipelineNode body
)
{
if (_allNode.TryGetValue(label, out PipelineNode? node)) {
throw new InvalidOperationException("Label has exist in scheduler.");
}
node = new PipelineNode(label, body, false);
_allNode.Add(label, node);
return node;
}
public PipelineNode GetNode(
PipelineNodeLabel label
)
{
if (_allNode.TryGetValue(label, out PipelineNode? node))
{
return node;
} else
{
throw new KeyNotFoundException();
}
}
public void LinkToInput(
PipelineNodeInPort port
)
{
ArgumentNullException.ThrowIfNull(port);
PipelineNodeOutPort p = _beginning.DefaultOut;
p.Connect(port);
}
public void LinkToInput(
PipelineNodeInPort port,
string inputPortName
)
{
ArgumentNullException.ThrowIfNull(port);
PipelineNodeOutPort p = _beginning.DefineCertainOutputPort(inputPortName);
p.Connect(port);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void SetInput(
params (string label, IPipeLineConnectionPayload arg)[] parameters
)
{
_beginning.SetInputData(parameters);
}
public void Terminate()
{
_concurrentLock.Wait();
(mtLevel, cLevel, mtIndex, cIndex) = (int.MaxValue, int.MaxValue, int.MaxValue, int.MaxValue);
_concurrentLock.Release();
}
public bool TryGetAllNode(
out List<List<PipelineNode>> mainThreadNodes,
out List<List<PipelineNode>> concurrentNodes
)
{
if (_isAvailable)
{
mainThreadNodes = _mainThreadNodes;
concurrentNodes = _concurrentNodes;
return true;
}
mainThreadNodes = null;
concurrentNodes = null;
return false;
}
internal bool RequestNexConcurrentNode(
out PipelineNode node
)
{
int index, level;
if (cLevel >= _concurrentNodes.Count)
{
node = null!;
return false;
}
_concurrentLock.Wait();
while (cIndex >= _concurrentNodes[cLevel].Count)
{
Interlocked.Increment(ref cLevel);
Interlocked.Exchange(ref cIndex, 0);
if (cLevel >= _concurrentNodes.Count)
{
node = null!;
return false;
}
}
index = cIndex;
level = cLevel;
Interlocked.Increment(ref cIndex);
_concurrentLock.Release();
node = _concurrentNodes[level][index];
return true;
}
internal bool RequestNextMainThreadNode(
out PipelineNode node
)
{
if (mtLevel >= _mainThreadNodes.Count)
{
node = null!;
return false;
}
while (mtIndex >= _mainThreadNodes[cLevel].Count)
{
Interlocked.Increment(ref mtLevel);
Interlocked.Exchange(ref mtIndex, 0);
if (mtLevel >= _mainThreadNodes.Count)
{
node = null!;
return false;
}
}
node = _mainThreadNodes[mtLevel][mtIndex];
return true;
}
internal void ResetIndex()
{
(mtLevel, cLevel, mtIndex, cIndex) = (0, 0, 0, 0);
_concurrentLock.Release();
}
void IConvertableToPipelineNode.NodeMain(
in Dictionary<string, PipelineNodeInPort> input,
in Dictionary<string, PipelineNodeOutPort> output
)
{
PipeLineRunner.Run(this);
PipelineNodeOutPort.SkipAll(output);
}
~PipelineScheduler()
{
runningStat.Dispose();
}
}
}
+112
View File
@@ -0,0 +1,112 @@
// -----------------------------------------------------------------------------
// file="PortArgs"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Security.Authentication.ExtendedProtection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
public class PipelineConnector
{
private protected IPipeLineConnectionPayload args;
private protected PipelineNodeInPort output;
private protected PipelineNodeOutPort input;
private protected SemaphoreSlim formerCompleteSemaphore;
public PipelineConnector(
PipelineNodeOutPort former,
PipelineNodeInPort after
)
{
this.input = former;
this.output = after;
formerCompleteSemaphore = new SemaphoreSlim(initialCount:0, maxCount:1);
}
public IPipeLineConnectionPayload Args
{
get => args;
set => args = value;
}
public PipelineNodeInPort Output => output;
public PipelineNodeOutPort Input => input;
public static void SkipAll(
Dictionary<string, PipelineConnector> ports
)
{
if (ports == null) {
return;
}
foreach (KeyValuePair<string, PipelineConnector> item in ports)
{
if (item.Value != null) {
item.Value.CompleteAndSkip();
}
}
}
public static void WaitAll(
Dictionary<string, PipelineConnector> ports
)
{
if (ports == null) {
return;
}
foreach (KeyValuePair<string, PipelineConnector> item in ports)
{
if (item.Value != null) {
item.Value.WaitForStart();
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public virtual void WaitForStart()
{
// Console.WriteLine($"Waiting \t{input.ContextName}\t->\t{output.ContextName}");
formerCompleteSemaphore.Wait();
}
~PipelineConnector()
{
formerCompleteSemaphore.Dispose();
}
#region pipline op
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompleteAndSkip()
{
// Console.WriteLine($"Complete\t{input.Name}\t->\t{output.Name}");
formerCompleteSemaphore.Release();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void CompleteAndWriteData(
IPipeLineConnectionPayload args
)
{
this.args = args;
CompleteAndSkip();
}
#endregion
}
public abstract class IPipeLineConnectionPayload { }
}
+79
View File
@@ -0,0 +1,79 @@
// -----------------------------------------------------------------------------
// file="SideNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Security.Principal;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.PipeLine
{
internal sealed class BeginningNode : PipelineNode
{
public const string OutName = "default";
private PipelineScheduler _bp;
public BeginningNode(
PipelineScheduler bp
)
: base(DefaultOperation, new PipelineNodeStringLabel("BeginningNode"), true,
["default"], [OutName])
{
_bp = bp;
}
public PipelineNodeOutPort DefaultOut => OutputPorts[OutName];
public override void Run()
{
_bp.RunningStat.Wait();
PipelineNodeOutPort.SkipAll(_output);
}
internal void SetInputData(
IEnumerable<(string, IPipeLineConnectionPayload)> data
)
{
if (data is null) {
return;
}
foreach ((string, IPipeLineConnectionPayload) item in data) {
OutputPorts[item.Item1].TransmitValue(item.Item2);
}
}
}
internal sealed class EndingNode : PipelineNode
{
private PipelineScheduler _bp;
public EndingNode(
PipelineScheduler bp
)
: base(DefaultOperation, name:new PipelineNodeStringLabel("EndingNode"), true,
["default"], ["default"])
{
_bp = bp;
}
public override void Run()
{
PipelineNodeInPort.WaitAll(_input);
_bp.ResetIndex();
_bp.RunningStat.Release();
}
}
}
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- https://go.microsoft.com/fwlink/?LinkID=208121. -->
<Project>
<PropertyGroup>
<Configuration>Release</Configuration>
<Platform>x64</Platform>
<PublishDir>C:\Users\Nitlo\OneDrive - MSFT\桌面\新建文件夹</PublishDir>
<PublishProtocol>FileSystem</PublishProtocol>
<_TargetId>Folder</_TargetId>
<TargetFramework>net8.0</TargetFramework>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<SelfContained>true</SelfContained>
<PublishSingleFile>true</PublishSingleFile>
<PublishReadyToRun>true</PublishReadyToRun>
<PublishTrimmed>true</PublishTrimmed>
</PropertyGroup>
</Project>
@@ -0,0 +1,36 @@
// -----------------------------------------------------------------------------
// file="ISimpleStateMachine"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.State
{
public interface ISimpleStateMachine<TStateNode, TStateLabel, TStateContext>
where TStateNode : SimpleStateMachineNode<TStateNode, TStateLabel, TStateContext>
where TStateLabel : unmanaged, IEquatable<TStateLabel>
where TStateContext : StateContext
{
TStateNode this[
TStateLabel label
] { get; }
ref readonly TStateNode GetCurrentStateNode();
void Reset();
bool TryMoveNext(
TStateContext ctx
);
}
}
@@ -0,0 +1,40 @@
// -----------------------------------------------------------------------------
// file="SimpleStateMachineNode"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.State
{
public abstract class SimpleStateMachineNode<TSelf, TLabel, TStateContext>
where TSelf : SimpleStateMachineNode<TSelf, TLabel, TStateContext>
where TLabel : unmanaged, IEquatable<TLabel>
where TStateContext : StateContext
{
private readonly List<int> _jumpTable = [];
public ReadOnlySpan<int> AvailableNodes => CollectionsMarshal.AsSpan(_jumpTable);
public abstract bool IsValidateFrom(
TSelf former,
TStateContext context
);
public abstract bool TryTransaction(
ReadOnlySpan<TSelf> nodeSpan,
TStateContext context,
out TLabel label
);
}
}
+17
View File
@@ -0,0 +1,17 @@
// -----------------------------------------------------------------------------
// file="StateContext"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.State
{
public abstract class StateContext { }
}
@@ -0,0 +1,18 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Library</OutputType>
<IsNativeHost>true</IsNativeHost>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
</ItemGroup>
<ItemGroup>
<Compile Remove="Graphics\**" />
<EmbeddedResource Remove="Graphics\**" />
<None Remove="Graphics\**" />
</ItemGroup>
</Project>
@@ -0,0 +1,347 @@
// -----------------------------------------------------------------------------
// file="AdaptiveBackoffStrategy"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Threading;
/// <summary>
/// Efficient adaptive backoff strategy that dynamically selects optimal backoff methods based on
/// system load and success rate
/// </summary>
public struct AdaptiveBackoffStrategy
{
private const int EVALUATION_INTERVAL_MS = 2000; // Global re-evaluation interval
private const int EXPLORATION_PROBABILITY = 10; // 10% probability to explore non-optimal strategies
private const int FAST_CONVERGENCE_THRESHOLD = 1000; // Fast convergence threshold
// Configuration constants
private const int INITIAL_SPINWAIT_THRESHOLD = 8; // Cheap spin count for first few SpinWait iterations
private const int MIN_SAMPLES_FOR_EVALUATION = 100; // Minimum sample count before starting evaluation
// Global shared state (all threads share learning results)
private static readonly PerformanceStats[] s_globalStats = new PerformanceStats[3];
private static volatile BackoffMethod s_globalBestMethod;
private static volatile int s_evaluationInProgress;
private static long s_lastGlobalEvaluationTicks;
private BackoffMethod _currentMethod;
private int _consecutiveFailures;
private int _localAttempts;
// Instance state
private SpinWait _spinWait;
private uint _fastRng; // Lightweight pseudo-random number generator
// Static constructor: initialize optimal strategy based on system characteristics
static AdaptiveBackoffStrategy()
{
// Based on your benchmark results:
// - Low core count (<= 8): SpinWait has clear advantage
// - Medium core count (<= 16): Close performance, slightly favor SpinWait
// - High core count (>= 20): Sleep0 has significant advantage under high concurrency
int logicalCores = Environment.ProcessorCount;
if (logicalCores <= 8)
{
s_globalBestMethod = BackoffMethod.SpinWait;
} else if (logicalCores <= 16)
{
s_globalBestMethod = BackoffMethod.SpinWait; // Default conservative choice
} else
{
// High core count systems, based on your 64-thread benchmark results
s_globalBestMethod = BackoffMethod.Sleep0;
}
s_lastGlobalEvaluationTicks = Environment.TickCount64;
// Initialize statistics array
for (int i = 0; i < s_globalStats.Length; i++)
{
s_globalStats[i].Reset();
}
}
/// <summary>
/// Gets the currently used backoff method (for debugging/monitoring)
/// </summary>
public readonly BackoffMethod CurrentMethod => _currentMethod;
/// <summary>
/// Gets the globally considered optimal backoff method
/// </summary>
public static BackoffMethod GlobalBestMethod => s_globalBestMethod;
/// <summary>
/// Creates a new adaptive backoff strategy instance
/// </summary>
public static AdaptiveBackoffStrategy Create()
{
return new AdaptiveBackoffStrategy
{
_spinWait = default,
_currentMethod = s_globalBestMethod,
_consecutiveFailures = 0,
_localAttempts = 0,
_fastRng =
(uint)(Environment.TickCount64 ^ Thread.CurrentThread.ManagedThreadId * 0x9E3779B9U)
};
}
/// <summary>
/// Gets global statistics for the specified strategy (for monitoring)
/// </summary>
public static (long attempts, long successes, double successRate) GetGlobalStats(
BackoffMethod method
)
{
PerformanceStats stats = s_globalStats[(int)method];
return (stats.Attempts, stats.Successes, stats.SuccessRate);
}
/// <summary>
/// Called after CAS failure, performs backoff and may switch strategies
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void OnFailureAndBackoff()
{
_consecutiveFailures++;
// Record failure
s_globalStats[(int)_currentMethod].RecordAttempt(success:false);
// Periodically re-evaluate global optimal strategy
if (ShouldEvaluateGlobalStrategy())
{
EvaluateAndUpdateGlobalStrategy();
}
// Decide whether to switch strategies based on failure count and current strategy
if (_consecutiveFailures >= INITIAL_SPINWAIT_THRESHOLD && ShouldSwitchStrategy())
{
SwitchToOptimalStrategy();
}
// Perform the actual backoff operation
PerformBackoff();
}
/// <summary>
/// Called after CAS success, resets state
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void OnSuccess()
{
// Record success
s_globalStats[(int)_currentMethod].RecordAttempt(success:true);
// Reset state
_spinWait = default;
_consecutiveFailures = 0;
// Fast convergence: tend to keep current strategy when continuously successful
if (_localAttempts < FAST_CONVERGENCE_THRESHOLD)
{
_localAttempts++;
}
}
/// <summary>
/// Evaluates and updates the global optimal strategy
/// </summary>
private static void EvaluateAndUpdateGlobalStrategy()
{
// Use CAS to ensure only one thread performs evaluation
if (Interlocked.CompareExchange(ref s_evaluationInProgress, 1, 0) != 0)
{
return;
}
try
{
long now = Environment.TickCount64;
// Double-check to avoid duplicate evaluation
if (now - Volatile.Read(ref s_lastGlobalEvaluationTicks) <= EVALUATION_INTERVAL_MS)
{
return;
}
BackoffMethod bestMethod = s_globalBestMethod;
double bestSuccessRate = -1.0;
// Find the strategy with the highest success rate
for (int i = 0; i < s_globalStats.Length; i++)
{
PerformanceStats stats = s_globalStats[i];
if (stats.Attempts >= MIN_SAMPLES_FOR_EVALUATION)
{
double successRate = stats.SuccessRate;
if (successRate > bestSuccessRate)
{
bestSuccessRate = successRate;
bestMethod = (BackoffMethod)i;
}
}
}
// Update global optimal strategy
s_globalBestMethod = bestMethod;
Volatile.Write(ref s_lastGlobalEvaluationTicks, now);
// Reset statistics (maintain exploratory behavior)
for (int i = 0; i < s_globalStats.Length; i++)
{
s_globalStats[i].Reset();
}
}
finally
{
Volatile.Write(ref s_evaluationInProgress, 0);
}
}
/// <summary>
/// Fast pseudo-random number generator (LCG)
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private uint NextRandom()
{
_fastRng = _fastRng * 1103515245U + 12345U;
return _fastRng;
}
/// <summary>
/// Performs the actual backoff operation
/// </summary>
[MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce hot path code size
private void PerformBackoff()
{
switch (_currentMethod)
{
case BackoffMethod.SpinWait:
if (_consecutiveFailures <= INITIAL_SPINWAIT_THRESHOLD)
{
_spinWait.SpinOnce(sleep1Threshold:-1);
} else if (_spinWait.NextSpinWillYield)
{
int yields = (int)(NextRandom() & 7) + 1; // 1-8 times
for (int i = 0; i < yields; i++) {
_ = Thread.Yield();
}
} else
{
_spinWait.SpinOnce(sleep1Threshold:-1);
}
break;
case BackoffMethod.Yield:
_ = Thread.Yield();
break;
case BackoffMethod.Sleep0:
Thread.Sleep(0);
break;
default:
break;
}
}
/// <summary>
/// Checks whether global strategy evaluation should be performed
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool ShouldEvaluateGlobalStrategy()
{
long now = Environment.TickCount64;
return now - Volatile.Read(ref s_lastGlobalEvaluationTicks) > EVALUATION_INTERVAL_MS;
}
/// <summary>
/// Determines whether strategy should be switched
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool ShouldSwitchStrategy()
{
// Fast convergence phase: reduce strategy switching
if (_localAttempts < FAST_CONVERGENCE_THRESHOLD && _consecutiveFailures < 32)
{
return false;
}
// Small probability exploration: occasionally try other strategies even when current is optimal
return NextRandom() % 100 < EXPLORATION_PROBABILITY || _currentMethod != s_globalBestMethod;
}
/// <summary>
/// Switches to the currently considered optimal strategy
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SwitchToOptimalStrategy()
{
BackoffMethod targetMethod;
// Most of the time choose the globally optimal
if (NextRandom() % 100 >= EXPLORATION_PROBABILITY)
{
targetMethod = s_globalBestMethod;
} else
{
// Small probability random exploration
targetMethod = (BackoffMethod)(NextRandom() % 3);
}
if (targetMethod != _currentMethod)
{
_currentMethod = targetMethod;
_spinWait = default; // Reset SpinWait state when switching strategies
_localAttempts = 0;
}
}
public enum BackoffMethod : byte
{
SpinWait = 0,
Yield = 1,
Sleep0 = 2
}
private struct PerformanceStats
{
public long Attempts;
public long LastResetTicks;
public long Successes;
public readonly double SuccessRate => Attempts == 0 ? 0.0 : (double)Successes / Attempts;
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void RecordAttempt(
bool success
)
{
_ = Interlocked.Increment(ref Attempts);
if (success) {
_ = Interlocked.Increment(ref Successes);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Reset()
{
Attempts = 0;
Successes = 0;
LastResetTicks = Environment.TickCount64;
}
}
}
@@ -0,0 +1,101 @@
// -----------------------------------------------------------------------------
// file="BufferExpansionLock"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Threading
{
/// <summary>
/// Provide a lock for managing concurrent access to a buffer that can be expanded and use a
/// common two-pase-copy expanding mechanism.
/// </summary>
public class BufferExpansionLock : IDisposable
{
private static readonly ThreadLocal<int> Count = new(() => 0);
private readonly ReaderWriterLockSlim _copyLock;
private readonly ReaderWriterLockSlim _expansionLock;
public BufferExpansionLock()
{
_expansionLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
_copyLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
}
public bool IsThreadReadingMetaData => _expansionLock.IsReadLockHeld;
public void Dispose()
{
_expansionLock?.Dispose();
_copyLock?.Dispose();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void EnterBufferCopy()
{
_copyLock.EnterWriteLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void EnterBufferRead()
{
int c = Count.Value;
if (c == 0) {
_copyLock.EnterReadLock();
}
Count.Value = c + 1;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void EnterExpansionProcess()
{
_expansionLock.EnterWriteLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void EnterMetaDataRead()
{
_expansionLock.EnterReadLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ExitBufferCopy()
{
_copyLock.ExitWriteLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ExitBufferRead()
{
int c = Count.Value - 1;
Count.Value = c;
if (c == 0) {
_copyLock.ExitReadLock();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ExitExpansionProcess()
{
_expansionLock.ExitWriteLock();
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ExitMetaDataRead()
{
_expansionLock.ExitReadLock();
}
}
}
@@ -0,0 +1,135 @@
// -----------------------------------------------------------------------------
// file="CancellableTask"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Threading
{
public class CancellableTask : IDisposable
{
private bool disposedValue;
private CancellationTokenSource cts;
private protected Action<CancellationToken> _startup;
private protected Task _internalTask;
private protected CancellableTask()
{
cts = new CancellationTokenSource();
}
private protected CancellableTask(
Action<CancellationToken> action
)
{
_internalTask = new Task(InternalAction);
cts = new CancellationTokenSource();
}
public bool IsCompleted => _internalTask.IsCompleted;
public CancellationTokenSource Cts => cts;
public int Id => _internalTask.Id;
public void Cancel()
{
Cts.Cancel();
}
public void CancelAndWait()
{
Cts.Cancel();
_internalTask.Wait();
}
// ~CancellableTask()
// {
// Dispose(disposing: false);
// }
public void Dispose()
{
Dispose(disposing:true);
GC.SuppressFinalize(this);
}
public static CancellableTask Run(
Action<CancellationToken> startup
)
{
CancellableTask ret = new CancellableTask
{
_startup = startup
};
ret._internalTask = Task.Run(ret.InternalAction);
return ret;
}
public void Wait()
{
_internalTask.Wait();
}
public static void WaitAll(
IEnumerable<CancellableTask> tasks
)
{
IEnumerable<Task> convertedTasks = tasks.Select<CancellableTask, Task>(
ct => ct._internalTask);
Task.WaitAll(convertedTasks.ToArray());
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void InternalAction()
{
_startup(Cts.Token);
}
protected virtual void Dispose(
bool disposing
)
{
if (!disposedValue)
{
if (disposing)
{
if (IsCompleted)
{
cts.Dispose();
_internalTask.Dispose();
} else
{
throw new InvalidOperationException(
"Cannot dispose a cancellable task before completed");
}
}
disposedValue = true;
}
}
public static implicit operator Task(
CancellableTask task
)
{
return task._internalTask;
}
~CancellableTask()
{
Cts.Dispose();
}
}
}
@@ -0,0 +1,163 @@
// -----------------------------------------------------------------------------
// file="SingleConsumerBell"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Threading;
namespace StgSharp.Threading
{
/// <summary>
/// A single-consumer bell/signal primitive.
/// </summary>
/// <remarks>
/// <para>Producers can call <see cref="Set" /> multiple times to publish signals. The bell
/// keeps a pending count of not-yet-consumed signals. The consumer uses <see
/// cref="Wait(object, CancellationToken)" /> or <see cref="TryWait(object)" /> to consume
/// signals. </para> <para>When a producer increments the pending count from 0 to 1, the
/// internal event is set to wake the consumer. Consuming via <see cref="TryWait(object)" />
/// or <see cref="Wait(object, CancellationToken)" /> clears the entire pending count
/// (aggregate consumption). If new signals arrive while resetting, the event is set again to
/// avoid missed signals. </para> <para>Ownership: only the specified <c> owner </c> can call
/// the consumer APIs. </para>
/// </remarks>
public sealed class SingleConsumerBell : IDisposable
{
private volatile bool _disposed;
private int _pending; // not volatile: used with Interlocked/Volatile APIs to avoid CS0420
private readonly ManualResetEventSlim _evt;
private readonly object _owner;
/// <summary>
/// Initializes a new instance of <see cref="SingleConsumerBell" />.
/// </summary>
/// <param name="owner">
/// The unique consumer identity; subsequent calls to Wait/TryWait must pass the same
/// reference.
/// </param>
/// <param name="spinCount">
/// Spin count passed to the underlying <see cref="ManualResetEventSlim" />.
/// </param>
/// <exception cref="ArgumentNullException">
/// Thrown if <paramref name="owner" /> is null.
/// </exception>
public SingleConsumerBell(
object owner,
int spinCount = 100
)
{
_owner = owner ?? throw new ArgumentNullException(nameof(owner));
_evt = new ManualResetEventSlim(false, spinCount);
}
/// <summary>
/// Disposes the bell, sets the event to wake any waiting consumer, and releases
/// resources.
/// </summary>
public void Dispose()
{
_disposed = true;
_evt.Set();
_evt.Dispose();
}
/// <summary>
/// Publishes a signal by incrementing the not-yet-consumed count and returns the current
/// pending count.
/// </summary>
/// <remarks>
/// If the pending count transitions from 0 to 1, the event is set to wake the consumer.
/// </remarks>
/// <returns>
/// The current pending count including this call.
/// </returns>
public int Set()
{
if (_disposed) {
return Volatile.Read(ref _pending);
}
// Increment pending; if we transitioned 0 -> 1, set the event to wake the consumer
int newCount = Interlocked.Increment(ref _pending);
if (newCount == 1) {
_evt.Set();
}
return newCount;
}
/// <summary>
/// Attempts to consume any pending signals without blocking.
/// </summary>
/// <param name="consumer">
/// The consumer object; must be the same reference provided at construction.
/// </param>
/// <returns>
/// <c> true </c> if a signal was available (pending or event set); otherwise
/// <c>false</c>.
/// </returns>
/// <exception cref="InvalidOperationException">
/// Thrown if <paramref name="consumer" /> is not the owner.
/// </exception>
public bool TryWait(
object consumer
)
{
if (!ReferenceEquals(_owner, consumer)) {
throw new InvalidOperationException("This bell is owned by another consumer.");
}
if (!_evt.IsSet && Volatile.Read(ref _pending) == 0) {
return false;
}
// Aggregate consumption: clear pending and reset event
_ = Interlocked.Exchange(ref _pending, 0);
_evt.Reset();
// If concurrent Set happened during reset, re-set the event to avoid lost signals
if (Volatile.Read(ref _pending) != 0)
{
_evt.Set();
}
return true;
}
/// <summary>
/// Blocks until a signal is available, then consumes all pending signals.
/// </summary>
/// <param name="consumer">
/// The consumer object; must be the same reference provided at construction.
/// </param>
/// <param name="cancellationToken">
/// A token to cancel the wait.
/// </param>
/// <exception cref="InvalidOperationException">
/// Thrown if <paramref name="consumer" /> is not the owner.
/// </exception>
public void Wait(
object consumer,
CancellationToken cancellationToken = default
)
{
if (!ReferenceEquals(_owner, consumer)) {
throw new InvalidOperationException("This bell is owned by another consumer.");
}
_evt.Wait(cancellationToken);
// Consume this round of signals: clear pending and handle reset/concurrency
_ = Interlocked.Exchange(ref _pending, 0);
_evt.Reset();
if (Volatile.Read(ref _pending) != 0) {
_evt.Set();
}
}
}
}
@@ -0,0 +1,64 @@
// -----------------------------------------------------------------------------
// file="ThreadHelper"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.ProcessorAbstraction;
using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Threading;
namespace StgSharp.Threading
{
public static partial class ThreadHelper
{
private static readonly int mainThreadId = Environment.CurrentManagedThreadId;
private static readonly ThreadLocal<bool> isNotMainThread = new(
() => Environment.CurrentManagedThreadId != mainThreadId);
static ThreadHelper()
{
EmptyThread.Start();
}
/// <summary>
/// An individual thread doing nothing, and its <see cref="Thread.IsAlive" /> <see
/// langword="property" /> always returns false.
/// </summary>
public static Thread EmptyThread { get; } = new(() => { });
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void BindToLogicalCore(
LogicalCore core
)
{
SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId);
}
[Conditional("DEBUG")]
internal static void MainThreadAssert()
{
if (isNotMainThread.Value) {
throw new InvalidOperationException(
$"Attempt to call sensitive operation which can be only called from main thread from thread {Environment.CurrentManagedThreadId}.");
}
}
#pragma warning disable CA5393
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static partial void SetThreadAffinity(
uint globalCoreId,
uint numaNodeId
);
#pragma warning restore CA5393
}
}
@@ -0,0 +1,334 @@
// -----------------------------------------------------------------------------
// file="VariableCapacitySemaphore"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using System;
using System.Runtime.CompilerServices;
using System.Threading;
namespace StgSharp.Threading
{
/// <summary>
/// A counting semaphore with adjustable maximum capacity at runtime. Implemented using one or
/// more ManualResetEventSlim (MRES) gates and atomic counters to avoid kernel semaphore
/// overhead while supporting high-frequency Release/Wait patterns. Notes: - This is a
/// counting primitive (not a barrier). Permits accumulate up to <see cref="MaxPermits" />. -
/// Reducing <see cref="MaxPermits" /> does not revoke already issued permits; it only clamps
/// future releases. - Striped MRES can reduce contention in highly parallel waits.
/// </summary>
public sealed class VariableCapacitySemaphore : IDisposable
{
// MRES stripes (one gate per stripe). All stripes are set when transitioning 0 -> >0 and reset at >0 -> 0.
private readonly ManualResetEventSlim[] _events;
private volatile int _current;
private volatile int _max;
private readonly int _stripeMask; // for power-of-two stripe counts; otherwise -1
/// <summary>
/// Create a new variable-capacity semaphore.
/// </summary>
/// <param name="initialCount">
/// Initial available permits (clamped to [0, maxPermits]).
/// </param>
/// <param name="maxPermits">
/// Logical maximum permits (>= 1).
/// </param>
/// <param name="stripes">
/// Optional stripe count for MRES (<= 0 to auto choose by CPU).
/// </param>
public VariableCapacitySemaphore(
int initialCount = 0,
int maxPermits = 1,
int stripes = 0
)
{
ArgumentOutOfRangeException.ThrowIfLessThan(maxPermits, 1);
ArgumentOutOfRangeException.ThrowIfNegative(initialCount);
_max = maxPermits;
_current = Math.Min(initialCount, maxPermits);
int stripeCount = (stripes > 0) ?
stripes :
ClampToPow2(Math.Min(Environment.ProcessorCount, 8));
_events = new ManualResetEventSlim[stripeCount];
bool set = _current > 0;
for (int i = 0; i < stripeCount; i++) {
_events[i] = new ManualResetEventSlim(set, 0);
}
_stripeMask = IsPow2(stripeCount) ? (stripeCount - 1) : (-1);
}
/// <summary>
/// Current number of available permits.
/// </summary>
public int CurrentCount => Volatile.Read(ref _current);
/// <summary>
/// Logical maximum number of permits. Can be changed at runtime.
/// </summary>
public int MaxPermits
{
get => Volatile.Read(ref _max);
set
{
ArgumentOutOfRangeException.ThrowIfLessThan(value, 1);
Volatile.Write(ref _max, value);
// Clamp current to not exceed the new max (do not block; let Wait() consume down).
while (true)
{
int cur = Volatile.Read(ref _current);
if (cur <= value)
{
break;
}
if (Interlocked.CompareExchange(ref _current, value, cur) == cur)
{
break;
}
}
// Reflect availability to gates
if (Volatile.Read(ref _current) > 0)
{
SetAll();
} else
{
ResetAll();
}
}
}
public void Dispose()
{
foreach (ManualResetEventSlim e in _events) {
e.Dispose();
}
}
/// <summary>
/// Release a single permit if under <see cref="MaxPermits" />. Returns the previous
/// count.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int Release()
{
while (true)
{
int cur = Volatile.Read(ref _current);
int max = Volatile.Read(ref _max);
if (cur >= max) {
return cur;
}
if (Interlocked.CompareExchange(ref _current, cur + 1, cur) == cur)
{
if (cur == 0) {
SetAll();
}
return cur;
}
}
}
/// <summary>
/// Release multiple permits (capped by <see cref="MaxPermits" />). Returns the previous
/// count.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public int Release(
int permits
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(permits);
while (true)
{
int cur = Volatile.Read(ref _current);
int max = Volatile.Read(ref _max);
if (cur >= max) {
return cur;
}
long target = ((long)cur) + permits;
if (target > max) {
target = max;
}
if (Interlocked.CompareExchange(ref _current, (int)target, cur) == cur)
{
if (cur == 0) {
SetAll();
}
return cur;
}
}
}
/// <summary>
/// Try to take a permit without blocking. Returns true if acquired.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public bool TryWait()
{
while (true)
{
int cur = Volatile.Read(ref _current);
if (cur == 0) {
return false;
}
if (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur)
{
if (cur - 1 == 0) {
ResetAll();
}
return true;
}
}
}
/// <summary>
/// Block until a permit is available, then consume one.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void Wait()
{
ManualResetEventSlim gate = GetStripeEvent();
SpinWait spinner = new SpinWait();
while (true)
{
int cur = Volatile.Read(ref _current);
if ((cur > 0) && (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur))
{
if (cur - 1 == 0) {
ResetAll();
}
return;
}
if (!spinner.NextSpinWillYield)
{
spinner.SpinOnce();
} else
{
gate.Wait();
}
}
}
/// <summary>
/// Block until a permit is available or canceled, then consume one.
/// </summary>
public bool Wait(
int millisecondsTimeout,
CancellationToken cancellationToken = default
)
{
ManualResetEventSlim gate = GetStripeEvent();
SpinWait spinner = new SpinWait();
int start = Environment.TickCount;
int remaining = millisecondsTimeout;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
int cur = Volatile.Read(ref _current);
if ((cur > 0) && (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur))
{
if (cur - 1 == 0) {
ResetAll();
}
return true;
}
if (!spinner.NextSpinWillYield)
{
spinner.SpinOnce();
continue;
}
if (millisecondsTimeout < 0)
{
gate.Wait(cancellationToken);
continue;
}
if (remaining <= 0) {
return false;
}
if (!gate.Wait(remaining, cancellationToken)) {
return false;
}
remaining = millisecondsTimeout - (Environment.TickCount - start);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static int ClampToPow2(
int x
)
{
if (x < 1) {
x = 1;
}
if (x > 8) {
x = 8;
}
int p = 1;
while (p << 1 <= x) {
p <<= 1;
}
return p;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private ManualResetEventSlim GetStripeEvent()
{
int id = Environment.CurrentManagedThreadId & 0x7fffffff;
if (_stripeMask != -1) {
return _events[id & _stripeMask];
}
int idx = id % _events.Length;
return _events[idx];
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static bool IsPow2(
int x
)
{
return (x & (x - 1)) == 0;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ResetAll()
{
foreach (ManualResetEventSlim e in _events) {
e.Reset();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SetAll()
{
foreach (ManualResetEventSlim e in _events) {
e.Set();
}
}
}
}
+122
View File
@@ -0,0 +1,122 @@
// -----------------------------------------------------------------------------
// file="StgSharpTime"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Collections;
using StgSharp.Timing;
using System;
using System.Diagnostics;
using System.Threading;
namespace StgSharp
{
public static partial class World
{
public static TimeSourceProviderBase MainTimeProvider => StgSharpTime.OnlyInstance;
}
public sealed class StgSharpTime : TimeSourceProviderBase
{
private static readonly LinearBag<TimeSpanProvider> _subscribers = new();
private static readonly Stopwatch mainProvider = new();
private static Thread timeProvideThread;
private StgSharpTime() { }
// Use Stopwatch frequency as base tick frequency (ticks per second).
public override long Frequency => Stopwatch.Frequency;
public static StgSharpTime OnlyInstance { get; } = new StgSharpTime();
public override long GetCurrentTimeSpanTick()
{
return mainProvider.ElapsedTicks;
}
public sealed override void StartProvidingTime()
{
if (mainProvider.IsRunning) {
return;
}
mainProvider.Start();
timeProvideThread = new Thread(ProvideTime)
{
IsBackground = true,
Name = "StgSharpTime"
};
timeProvideThread.Start();
}
public override void StopProvidingTime()
{
lock (mainProvider) {
mainProvider.Stop();
}
}
public void Terminate()
{
lock (mainProvider) {
mainProvider.Stop();
}
}
protected sealed override void AddSubscriberUnsynced(
TimeSpanProvider subscriber
)
{
_subscribers.Add(subscriber);
}
protected sealed override void RemoveSubscriberUnsynced(
TimeSpanProvider subscriber
)
{
_subscribers.Remove(subscriber);
}
private void ProvideTime()
{
long internalFrequency = Stopwatch.Frequency; // ticks per second
lock (mainProvider) {
mainProvider.Restart();
}
while (mainProvider.IsRunning)
{
long elapsedTicks;
lock (mainProvider) {
elapsedTicks = mainProvider.ElapsedTicks;
}
if (_subscribers.Count == 0)
{
Thread.SpinWait(50);
continue;
}
lock (SubscribeLock)
{
for (int i = 0; i < _subscribers.Count; i++)
{
if (!_subscribers[i].OnSpanTick(elapsedTicks))
{
// Console.WriteLine("removed");
_subscribers.RemoveAt(i);
i--; // adjust index after removal
}
}
}
// Optional: brief spin to avoid tight loop if frame lengths are large.
Thread.SpinWait(20);
}
}
}
}
+301
View File
@@ -0,0 +1,301 @@
// -----------------------------------------------------------------------------
// file="TimeSequence"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Timing;
using System;
using System.Runtime.CompilerServices;
namespace StgSharp.Timing
{
public abstract class TimeSequence
{
public long BeginningTick { get; protected set; }
public long EndingTick { get; protected set; }
public long PreviousFrameTick { get; protected set; }
public static TimeSequence CreateLinear(
double totalSeconds,
double frameSeconds
)
{
if (double.IsPositiveInfinity(totalSeconds)) {
return new InfiniteLinearTimeSequence(frameSeconds);
}
return new LinearTimeSequence(totalSeconds, frameSeconds);
}
public static TimeSequence CreateLinearInfinite(
double frameSeconds
)
{
return new InfiniteLinearTimeSequence(frameSeconds);
}
public static TimeSequence CreateLogarithmic(
double initialSliceSeconds,
double totalSeconds,
double growthMultiplier,
double minSliceSeconds
)
{
return new LogTimeSequence(initialSliceSeconds, totalSeconds, growthMultiplier,
minSliceSeconds);
}
public abstract bool EndsAt(
long currentTick
);
public abstract bool IsNextFrameReady(
long currentTick
);
public abstract void StartFrom(
TimeSourceProviderBase source
);
}
internal sealed class LinearTimeSequence : TimeSequence
{
private readonly double _frameSeconds;
private readonly double _totalSeconds;
private long _frameLengthTicks;
public LinearTimeSequence(
double totalSeconds,
double frameSeconds
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(totalSeconds);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(frameSeconds);
if (frameSeconds > totalSeconds) {
throw new ArgumentException("Frame length greater than total length.");
}
_totalSeconds = totalSeconds;
_frameSeconds = frameSeconds;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool EndsAt(
long currentTick
)
{
return currentTick >= EndingTick;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsNextFrameReady(
long currentTick
)
{
long span = currentTick - PreviousFrameTick;
if (span >= _frameLengthTicks && currentTick < EndingTick)
{
PreviousFrameTick += _frameLengthTicks;
return true;
}
return false;
}
public override void StartFrom(
TimeSourceProviderBase source
)
{
long f = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
_frameLengthTicks = (long)Math.Max(1, _frameSeconds * f);
long totalTicks = (long)Math.Max(_frameLengthTicks, Math.Round(_totalSeconds * f, MidpointRounding.AwayFromZero));
BeginningTick = source.GetCurrentTimeSpanTick();
EndingTick = BeginningTick + totalTicks;
EndingTick = EndingTick < 0 ? long.MaxValue : EndingTick;
PreviousFrameTick = BeginningTick;
}
}
/// <summary>
/// Logarithmic time sequence with growth factor and minimum slice length enforcement.
/// </summary>
internal sealed class LogTimeSequence : TimeSequence
{
private bool _finalEmitted;
private readonly double _initialSeconds;
private double _lastBoundarySecondsAdjusted;
private readonly double _minSliceSeconds;
private readonly double _multiplier;
private double _nextBoundarySecondsAdjusted;
private readonly double _totalSeconds;
private int _sliceIndex;
private long _frequency;
private long _nextBoundaryTick;
private long _totalTicks;
public LogTimeSequence(
double initialSliceSeconds,
double totalSeconds,
double growthMultiplier,
double minSliceSeconds
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(initialSliceSeconds);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(totalSeconds);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(growthMultiplier);
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(minSliceSeconds);
if (growthMultiplier < 1.0) {
throw new ArgumentException("growthMultiplier must be >= 1");
}
if (initialSliceSeconds > totalSeconds) {
throw new ArgumentException("Initial slice length greater than total length.");
}
_initialSeconds = initialSliceSeconds;
_totalSeconds = totalSeconds;
_multiplier = growthMultiplier;
_minSliceSeconds = minSliceSeconds;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool EndsAt(
long currentTick
)
{
return currentTick >= EndingTick;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsNextFrameReady(
long currentTick
)
{
// Emit final once when reaching or passing end.
if (currentTick >= EndingTick)
{
if (_finalEmitted) {
return false;
}
PreviousFrameTick = EndingTick;
_finalEmitted = true;
return true;
}
// Not reached end: only when we cross the next boundary tick do we emit.
if (currentTick < _nextBoundaryTick)
{
return false;
}
// Advance to this boundary.
PreviousFrameTick = _nextBoundaryTick;
_lastBoundarySecondsAdjusted = _nextBoundarySecondsAdjusted;
_sliceIndex++;
// Compute next cumulative boundary seconds with multiplier, then enforce min-slice and total cap.
double next = _initialSeconds * Math.Pow(_multiplier, _sliceIndex);
if (next > _totalSeconds) {
next = _totalSeconds;
}
double delta = next - _lastBoundarySecondsAdjusted;
if (delta < _minSliceSeconds)
{
next = _lastBoundarySecondsAdjusted + _minSliceSeconds;
if (next > _totalSeconds) {
next = _totalSeconds;
}
}
_nextBoundarySecondsAdjusted = next;
_nextBoundaryTick = (_nextBoundarySecondsAdjusted >= _totalSeconds) ?
EndingTick :
BeginningTick + SecondsToTicks(_nextBoundarySecondsAdjusted);
return true;
}
public override void StartFrom(
TimeSourceProviderBase source
)
{
_frequency = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
BeginningTick = source.GetCurrentTimeSpanTick();
_totalTicks = SecondsToTicks(_totalSeconds);
EndingTick = BeginningTick + _totalTicks;
_sliceIndex = 0;
_lastBoundarySecondsAdjusted = 0.0;
_nextBoundarySecondsAdjusted = _initialSeconds;
_nextBoundaryTick = BeginningTick + SecondsToTicks(_nextBoundarySecondsAdjusted);
PreviousFrameTick = BeginningTick;
_finalEmitted = false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private long SecondsToTicks(
double seconds
)
{
return (long)Math.Max(1L, Math.Round(seconds * _frequency, MidpointRounding.AwayFromZero));
}
}
/// <summary>
/// Infinite linear time sequence: emits frames at fixed interval and never ends.
/// </summary>
internal sealed class InfiniteLinearTimeSequence : TimeSequence
{
private readonly double _frameSeconds;
private long _frameLengthTicks;
public InfiniteLinearTimeSequence(
double frameSeconds
)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(frameSeconds);
_frameSeconds = frameSeconds;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool EndsAt(
long currentTick
)
{
return false; // never ends
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public override bool IsNextFrameReady(
long currentTick
)
{
long span = currentTick - PreviousFrameTick;
if (span >= _frameLengthTicks)
{
PreviousFrameTick += _frameLengthTicks;
return true;
}
return false;
}
public override void StartFrom(
TimeSourceProviderBase source
)
{
long f = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
_frameLengthTicks = (long)Math.Max(1, _frameSeconds * f);
BeginningTick = source.GetCurrentTimeSpanTick();
EndingTick = long.MaxValue; // sentinel
PreviousFrameTick = BeginningTick;
}
}
}
@@ -0,0 +1,56 @@
// -----------------------------------------------------------------------------
// file="TimeSourceProvider"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp;
using System;
using System.Linq;
using System.Threading;
namespace StgSharp.Timing
{
public abstract class TimeSourceProviderBase
{
public abstract long Frequency { get; }
protected object SubscribeLock { get; } = new();
public void AddSubscriber(
TimeSpanProvider subscriber
)
{
lock (SubscribeLock) {
AddSubscriberUnsynced(subscriber);
}
}
public abstract long GetCurrentTimeSpanTick();
public void RemoveSubscriber(
TimeSpanProvider subscriber
)
{
lock (SubscribeLock) {
RemoveSubscriberUnsynced(subscriber);
}
}
public abstract void StartProvidingTime();
public abstract void StopProvidingTime();
protected abstract void AddSubscriberUnsynced(
TimeSpanProvider subscriber
);
protected abstract void RemoveSubscriberUnsynced(
TimeSpanProvider subscriber
);
}
}
@@ -0,0 +1,223 @@
// -----------------------------------------------------------------------------
// file="TimeSpanProvider.Awaiting"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.HighPerformance.Memory;
using System;
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace StgSharp.Timing
{
public partial class TimeSpanProvider
{
private static readonly ConcurrentBufferStack<int> s_unusedID = new ConcurrentBufferStack<int>(8);
private static int s_maxID;
private EventHandler? _refreshHandler, _missHandler;
private int _size, _id;
private long _pendingSpans; // Number of unconsumed span ticks since last Wait.
private readonly ManualResetEventSlim _event = new();
private readonly object _participantObject = new();
public event EventHandler MissTimeSpanRefreshed
{
add => _missHandler += value;
remove => _missHandler -= value;
}
public event EventHandler TimeSpanRefreshed
{
add => _refreshHandler += value;
remove => _refreshHandler -= value;
}
/// <summary>
/// True if there is at least one unconsumed span tick (PendingSpanCount &gt; 0).
/// </summary>
public bool HasPendingSpan => PendingSpanCount > 0;
public int TokenID
{
get => _id;
private set => _id = value;
}
/// <summary>
/// Number of span ticks not yet consumed by Wait ( &gt; 0 means last tick not waited).
/// </summary>
public long PendingSpanCount => Volatile.Read(ref _pendingSpans);
internal ManualResetEventSlim AwaitingEvent => _event;
public void Dispose()
{
// mark provider as ended so external users can observe terminal state
Volatile.Write(ref _endedFlag, 1);
s_unusedID.Push(TokenID);
_event.Set();
GC.SuppressFinalize(this);
}
public override int GetHashCode()
{
return _id;
}
public void Participant()
{
lock (_participantObject)
{
// Increase the maximum concurrent consumers for the next tick.
_size++;
}
}
public void WaitNextSpan()
{
_event.Wait();
// Consume one span slot (clamp at 0 so multiple subscribers won't make it negative).
while (true)
{
long cur = Volatile.Read(ref _pendingSpans);
if (cur == 0) {
return;
}
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
{
if (cur - 1 == 0)
{
// Last consumer resets the gate for the next tick.
_event.Reset();
}
return;
}
}
}
public async Task WaitNextSpanAsync()
{
await Task.Run(() => _event.Wait()).ConfigureAwait(false);
while (true)
{
long cur = Volatile.Read(ref _pendingSpans);
if (cur == 0)
{
break;
}
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
{
if (cur - 1 == 0) {
_event.Reset();
}
break;
}
}
}
/// <summary>
/// Wait for the next timespan tick, but honor the provided cancellation token. Throws
/// OperationCanceledException when token is cancelled. If the provider is disposed during
/// wait an OperationCanceledException is thrown to indicate end.
/// </summary>
public Task WaitNextSpanAsync(
CancellationToken ct
)
{
// Block the current thread until the event is signaled or cancellation requested.
_event.Wait(ct);
// consume one pending span (same logic as non-cancellable variant)
while (true)
{
long cur = Volatile.Read(ref _pendingSpans);
if (cur == 0)
{
break;
}
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
{
if (cur - 1 == 0) {
_event.Reset();
}
break;
}
}
return Task.CompletedTask;
}
internal void MissRefresh()
{
_missHandler?.Invoke(this, EventArgs.Empty);
} // Legacy compatibility.
/// <summary>
///
/// </summary>
/// <param name="currentTick">
/// Tick passed since binded time source.
/// </param>
/// <returns>
/// True if the span provider is working. False if time is over (the sequence ended).
/// </returns>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal bool OnSpanTick(
long currentTick
)
{
if (_sequence.EndsAt(currentTick))
{
// mark ended due to sequence timeout, then dispose
Volatile.Write(ref _endedFlag, 1);
// Console.WriteLine("Time span ended.");
Dispose();
return false;
}
if (!_sequence.IsNextFrameReady(currentTick)) {
return true;
}
// Console.WriteLine("Frame hit!");
// Fast path: if no pending consumers, initialize the counter to current capacity (_size).
if (Volatile.Read(ref _pendingSpans) == 0)
{
int toIssue = Math.Max(1, Volatile.Read(ref _size));
if (Interlocked.CompareExchange(ref _pendingSpans, toIssue, 0) == 0)
{
// Open the gate for all current waiters (up to `_size`).
_event.Set();
_refreshHandler?.Invoke(this, EventArgs.Empty);
}
} else
{
// A tick is already pending (or was just set by another thread): mark as miss.
_missHandler?.Invoke(this, EventArgs.Empty);
}
return true;
}
~TimeSpanProvider()
{
Dispose();
}
}
}
@@ -0,0 +1,83 @@
// -----------------------------------------------------------------------------
// file="TimeSpanProvider"
// Project: StgSharp
// Copyright (c) Nitload.
// SPDX-License-Identifier: MIT
// -----------------------------------------------------------------------------
using StgSharp.Timing;
using System;
using System.Runtime.CompilerServices;
using System.Threading;
namespace StgSharp.Timing
{
/// <summary>
/// TimeSpanProvider: uses a bound TimeSequence and TimeSourceProviderBase to publish span
/// ticks. Each OnSpanTick advances the sequence; when sequence ends provider unsubscribes.
/// </summary>
public partial class TimeSpanProvider
{
// private bool _subscribed;
private readonly int _emittedSpanCount; // legacy counter
// internal ended flag (0 = running, 1 = ended)
private int _endedFlag;
private readonly TimeSequence _sequence;
private readonly TimeSourceProviderBase _timeSource;
public TimeSpanProvider(
TimeSequence sequence,
TimeSourceProviderBase provider,
int maxWaitCount = 1
)
{
ArgumentNullException.ThrowIfNull(provider);
ArgumentNullException.ThrowIfNull(sequence);
_timeSource = provider;
_sequence = sequence;
_sequence.StartFrom(provider);
provider.AddSubscriber(this);
// _subscribed = true;
if (!s_unusedID.TryPop(out int id))
{
id = Interlocked.Increment(ref s_maxID);
}
TokenID = id;
_size = maxWaitCount > 0 ? maxWaitCount : 1;
_event = new ManualResetEventSlim(initialState:false, spinCount:0);
_pendingSpans = 0;
}
/// <summary>
/// True when the provider has reached its end and will not emit further spans.
/// </summary>
public bool IsEnded => Volatile.Read(ref _endedFlag) != 0;
/// <summary>
/// Current emitted time in seconds relative to the sequence beginning.
/// </summary>
public double CurrentSecond => (double)(_sequence.PreviousFrameTick - _sequence.BeginningTick) / _timeSource.Frequency;
public int CurrentSpan => _emittedSpanCount;
public static TimeSourceProviderBase DefaultProvider => World.MainTimeProvider;
public static int ToInt32(
TimeSpanProvider provider
)
{
return provider?._emittedSpanCount ?? 0;
}
public static explicit operator int(
TimeSpanProvider provider
)
{
return provider?._emittedSpanCount ?? 0;
}
}
}