Using new file structure. Use new regexir display method. Fix line dependency error in prefix analysing.
Signed-off-by: Nitload <YoumuLiu_NSI@whv0t.onmicrosoft.com>
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="BidirectionalDictionary"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Linq;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public sealed class BidirectionalDictionary<TFirst, TSecond> : IBidirectionalDictionary<TFirst, TSecond>
|
||||
where TFirst : notnull
|
||||
where TSecond : notnull
|
||||
{
|
||||
|
||||
private readonly Dictionary<TFirst, TSecond> _forward;
|
||||
private readonly Dictionary<TSecond, TFirst> _reverse;
|
||||
|
||||
public BidirectionalDictionary()
|
||||
{
|
||||
_forward = [];
|
||||
_reverse = [];
|
||||
}
|
||||
|
||||
public TFirst this[
|
||||
TSecond key
|
||||
]
|
||||
{
|
||||
get => _reverse.TryGetValue(key, out TFirst value) ?
|
||||
value :
|
||||
throw new ArgumentException($"Cannot find key {key} in dictionary.");
|
||||
set
|
||||
{
|
||||
if (_reverse.TryGetValue(key, out TFirst origin))
|
||||
{
|
||||
_reverse[key] = value;
|
||||
_ = _forward.Remove(origin);
|
||||
} else
|
||||
{
|
||||
_reverse.Add(key, value);
|
||||
}
|
||||
_forward.Add(value, key);
|
||||
}
|
||||
}
|
||||
|
||||
public TSecond this[
|
||||
TFirst key
|
||||
]
|
||||
{
|
||||
get => _forward.TryGetValue(key, out TSecond value) ?
|
||||
value :
|
||||
throw new ArgumentException($"Cannot find key {key} in dictionary.");
|
||||
set
|
||||
{
|
||||
if (_forward.TryGetValue(key, out TSecond origin))
|
||||
{
|
||||
_forward[key] = value;
|
||||
_ = _reverse.Remove(origin);
|
||||
} else
|
||||
{
|
||||
_forward.Add(key, value);
|
||||
}
|
||||
_reverse.Add(value, key);
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsReadOnly => false;
|
||||
|
||||
public ICollection<TFirst> FirstIndex => _forward.Keys;
|
||||
|
||||
public ICollection<TSecond> SecondIndex => _reverse.Keys;
|
||||
|
||||
public int Count => _forward.Count;
|
||||
|
||||
public IReadOnlyDictionary<TFirst, TSecond> Forward => new ReadOnlyDictionary<TFirst, TSecond>(
|
||||
_forward);
|
||||
|
||||
public IReadOnlyDictionary<TSecond, TFirst> Reverse => new ReadOnlyDictionary<TSecond, TFirst>(
|
||||
_reverse);
|
||||
|
||||
public void Add(
|
||||
KeyValuePair<TFirst, TSecond> item
|
||||
)
|
||||
{
|
||||
if (_forward.ContainsKey(item.Key) || _reverse.ContainsKey(item.Value)) {
|
||||
throw new ArgumentException("Duplicate key or value.");
|
||||
}
|
||||
_forward.Add(item.Key, item.Value);
|
||||
_reverse.Add(item.Value, item.Key);
|
||||
}
|
||||
|
||||
public void Add(
|
||||
TFirst key,
|
||||
TSecond value
|
||||
)
|
||||
{
|
||||
if (_forward.ContainsKey(key) || _reverse.ContainsKey(value)) {
|
||||
throw new ArgumentException("Duplicate key or value.");
|
||||
}
|
||||
_forward.Add(key, value);
|
||||
_reverse.Add(value, key);
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
_forward.Clear();
|
||||
_reverse.Clear();
|
||||
}
|
||||
|
||||
public bool Contains(
|
||||
KeyValuePair<TFirst, TSecond> item
|
||||
)
|
||||
{
|
||||
return _forward.Contains(item) &&
|
||||
_reverse.TryGetValue(item.Value, out TFirst var) &&
|
||||
var!.Equals(item.Key);
|
||||
}
|
||||
|
||||
public bool Contains(
|
||||
TSecond key
|
||||
)
|
||||
{
|
||||
return _reverse.ContainsKey(key);
|
||||
}
|
||||
|
||||
public bool Contains(
|
||||
TFirst key
|
||||
)
|
||||
{
|
||||
return _forward.ContainsKey(key);
|
||||
}
|
||||
|
||||
public void CopyTo(
|
||||
KeyValuePair<TFirst, TSecond>[] array,
|
||||
int arrayIndex
|
||||
)
|
||||
{
|
||||
if (array == null) {
|
||||
throw new ArgumentNullException(nameof(array));
|
||||
}
|
||||
if (arrayIndex < 0 || arrayIndex > array.Length) {
|
||||
throw new ArgumentOutOfRangeException(nameof(arrayIndex));
|
||||
}
|
||||
if (array.Length - arrayIndex < _forward.Count) {
|
||||
throw new ArgumentException("The array is too small to copy the elements.");
|
||||
}
|
||||
|
||||
foreach (KeyValuePair<TFirst, TSecond> pair in _forward) {
|
||||
array[arrayIndex++] = pair;
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerator<KeyValuePair<TFirst, TSecond>> GetEnumerator()
|
||||
{
|
||||
foreach (KeyValuePair<TFirst, TSecond> pair in _forward) {
|
||||
yield return pair;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Remove(
|
||||
TFirst key
|
||||
)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public bool Remove(
|
||||
KeyValuePair<TFirst, TSecond> item
|
||||
)
|
||||
{
|
||||
if (_forward.Remove(item.Key)) {
|
||||
return _reverse.Remove(item.Value);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool TryGetValue(
|
||||
TFirst key,
|
||||
out TSecond value
|
||||
)
|
||||
{
|
||||
return _forward.TryGetValue(key, out value);
|
||||
}
|
||||
|
||||
public bool TryGetValue(
|
||||
TSecond key,
|
||||
out TFirst value
|
||||
)
|
||||
{
|
||||
return _reverse.TryGetValue(key, out value);
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="CapacityFixedBag"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Reflection;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public sealed class CapacityFixedBag<T> : IEnumerable<T>
|
||||
{
|
||||
|
||||
private readonly T[] _buffer;
|
||||
|
||||
public CapacityFixedBag(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
_buffer = new T[capacity];
|
||||
}
|
||||
|
||||
public int Count { get; set; }
|
||||
|
||||
public int Capacity => _buffer.Length;
|
||||
|
||||
public void Add(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (Count >= _buffer.Length) {
|
||||
throw new InvalidOperationException("Bag capacity exceeded.");
|
||||
}
|
||||
_buffer[Count] = item;
|
||||
Count++;
|
||||
}
|
||||
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
for (int i = 0; i < Count; i++) {
|
||||
yield return _buffer[i];
|
||||
}
|
||||
}
|
||||
|
||||
public void Remove(
|
||||
T item
|
||||
)
|
||||
{
|
||||
int index = Array.IndexOf(_buffer, item, 0, Count);
|
||||
if (index >= 0)
|
||||
{
|
||||
_buffer[index] = _buffer[Count - 1];
|
||||
_buffer[Count - 1] = default!;
|
||||
Count--;
|
||||
}
|
||||
}
|
||||
|
||||
public void RemoveAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
if (index >= 0)
|
||||
{
|
||||
_buffer[index] = _buffer[Count - 1];
|
||||
_buffer[Count - 1] = default!;
|
||||
Count--;
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="CapacityFixedList"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public static class CapacityFixedListBuilder
|
||||
{
|
||||
|
||||
internal static CapacityFixedList<T> Create<T>(
|
||||
ReadOnlySpan<T> values
|
||||
)
|
||||
{
|
||||
return new CapacityFixedList<T>(values);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[CollectionBuilder(typeof(CapacityFixedListBuilder), nameof(CapacityFixedListBuilder.Create))]
|
||||
public class CapacityFixedList<T> : IList<T>
|
||||
{
|
||||
|
||||
private T[] _values;
|
||||
private readonly int _capacity;
|
||||
private int _index;
|
||||
|
||||
public CapacityFixedList(
|
||||
int size
|
||||
)
|
||||
{
|
||||
_values = new T[size];
|
||||
_capacity = size;
|
||||
}
|
||||
|
||||
public CapacityFixedList(
|
||||
ReadOnlySpan<T> values
|
||||
)
|
||||
{
|
||||
_values = values.ToArray();
|
||||
}
|
||||
|
||||
public T this[
|
||||
int index
|
||||
]
|
||||
{
|
||||
get => index < _index ?
|
||||
_values[index] :
|
||||
throw new ArgumentOutOfRangeException(nameof(index));
|
||||
set => _values[index] =
|
||||
index < _index ? value : throw new ArgumentOutOfRangeException(nameof(index));
|
||||
}
|
||||
|
||||
public bool IsReadOnly => false;
|
||||
|
||||
public int Count => _index + 1;
|
||||
|
||||
public void Add(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (_index < _capacity - 1)
|
||||
{
|
||||
_index++;
|
||||
this[_index] = item;
|
||||
}
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
|
||||
Array.Fill(_values, default);
|
||||
}
|
||||
_index = 0;
|
||||
}
|
||||
|
||||
public bool Contains(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (item is null) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i <= _index; i++)
|
||||
{
|
||||
if (item.Equals(_values[i])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void CopyTo(
|
||||
T[] array,
|
||||
int arrayIndex
|
||||
)
|
||||
{
|
||||
_values.CopyTo(array, arrayIndex);
|
||||
}
|
||||
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
for (int i = 0; i <= _index; i++) {
|
||||
yield return _values[i];
|
||||
}
|
||||
}
|
||||
|
||||
public int IndexOf(
|
||||
T item
|
||||
)
|
||||
{
|
||||
T v;
|
||||
for (int i = 0; i <= _index; i++)
|
||||
{
|
||||
if ((v = _values[i]) != null && v.Equals(item)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
public bool Remove(
|
||||
T item
|
||||
)
|
||||
{
|
||||
T v;
|
||||
for (int i = 0; i <= _index; i++)
|
||||
{
|
||||
if ((v = _values[i]) != null && v.Equals(item))
|
||||
{
|
||||
_values[i] = _values[_index];
|
||||
_index--;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void RemoveAll(
|
||||
Predicate<T> predicate
|
||||
)
|
||||
{
|
||||
if (predicate is null) {
|
||||
return;
|
||||
}
|
||||
for (int i = _index; i > 0; i--)
|
||||
{
|
||||
if (predicate(_values[i]))
|
||||
{
|
||||
_values[i] = _values[_index];
|
||||
_index--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void RemoveAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
if (index <= _index)
|
||||
{
|
||||
Array.Copy(_values, index + 1, _values, index, _index - index);
|
||||
if (RuntimeHelpers.IsReferenceOrContainsReferences<T>()) {
|
||||
_values[_index] = default!;
|
||||
}
|
||||
_index--;
|
||||
}
|
||||
}
|
||||
|
||||
public void Resize(
|
||||
int newSize
|
||||
)
|
||||
{
|
||||
Array.Resize(ref _values, newSize);
|
||||
}
|
||||
|
||||
internal Span<T> ToSpan()
|
||||
{
|
||||
return _values;
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
yield return GetEnumerator();
|
||||
}
|
||||
|
||||
void IList<T>.Insert(
|
||||
int index,
|
||||
T item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
}//------------------------------------- End of Class ------------------------------------------
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="CapacityFixedStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
[CollectionBuilder(typeof(CapacityFixedStackBuilder), nameof(CapacityFixedStackBuilder.Create))]
|
||||
public class CapacityFixedStack<T> : IEnumerable<T>
|
||||
{
|
||||
|
||||
private readonly T[] _values;
|
||||
|
||||
// _index points to the next free slot (top empty position). Therefore Count == _index.
|
||||
private int _index;
|
||||
|
||||
public CapacityFixedStack(
|
||||
int size
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(size);
|
||||
_values = new T[size];
|
||||
_index = 0; // empty stack
|
||||
}
|
||||
|
||||
public T? First => Volatile.Read(ref _index) > 0 ? _values[0] : default;
|
||||
|
||||
public bool IsEmpty => Volatile.Read(ref _index) == 0;
|
||||
|
||||
public int Count => Volatile.Read(ref _index);
|
||||
|
||||
public int Capacity => _values.Length;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ReadOnlySpan<T> AsSpan()
|
||||
{
|
||||
return _values.AsSpan(0, _index);
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
if (_index > 0)
|
||||
{
|
||||
Array.Clear(_values, 0, _index);
|
||||
_index = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
for (int i = _index - 1; i >= 0; i--) {
|
||||
yield return _values[i];
|
||||
}
|
||||
}
|
||||
|
||||
public T Peek()
|
||||
{
|
||||
if (_index == 0) {
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
return _values[_index - 1];
|
||||
}
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
if (_index == 0) {
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
_index--;
|
||||
T item = _values[_index];
|
||||
_values[_index] = default!; // Clear reference for GC
|
||||
return item;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T[] PopRange(
|
||||
int count
|
||||
)
|
||||
{
|
||||
if (count <= 0 || _index == 0) {
|
||||
return Array.Empty<T>();
|
||||
}
|
||||
int n = Math.Min(count, _index);
|
||||
int start = _index - n;
|
||||
T[] ret = new T[n];
|
||||
Array.Copy(_values, start, ret, 0, n);
|
||||
Array.Clear(_values, start, n);
|
||||
_index -= n;
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void Push(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (_index >= _values.Length) {
|
||||
throw new InvalidOperationException("Stack is full.");
|
||||
}
|
||||
_values[_index] = item;
|
||||
_index++;
|
||||
}
|
||||
|
||||
public void PushRange(
|
||||
ReadOnlySpan<T> span
|
||||
)
|
||||
{
|
||||
if (span.Length == 0) {
|
||||
return;
|
||||
}
|
||||
if (_index + span.Length > _values.Length) {
|
||||
throw new InvalidOperationException("Not enough space in the stack to push the entire span.");
|
||||
}
|
||||
span.CopyTo(_values.AsSpan(_index, span.Length));
|
||||
_index += span.Length;
|
||||
}
|
||||
|
||||
public bool TryPop(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
if (_index == 0)
|
||||
{
|
||||
value = default!;
|
||||
return false;
|
||||
}
|
||||
_index--;
|
||||
value = _values[_index];
|
||||
_values[_index] = default!;
|
||||
return true;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal Span<T> GetValueLayout()
|
||||
{
|
||||
return _values.AsSpan();
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class CapacityFixedStackBuilder
|
||||
{
|
||||
|
||||
public static CapacityFixedStack<T> Create<T>(
|
||||
ReadOnlySpan<T> values
|
||||
)
|
||||
{
|
||||
CapacityFixedStack<T> stack = new(values.Length);
|
||||
stack.PushRange(values); // Push in order
|
||||
return stack;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ConcurrentCapacityFixedStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Lock-free, fixed-capacity stack implemented on top of an array using atomic index
|
||||
/// operations. - No linked list; only array slots - Uses CAS on a shared top index to
|
||||
/// reserve/release slots - Per-slot state flag to safely publish/consume values under races
|
||||
/// </summary>
|
||||
public class ConcurrentCapacityFixedStack<T> : IEnumerable<T>
|
||||
{
|
||||
|
||||
private readonly int[] _states; // 0 = empty, 1 = full
|
||||
private readonly T[] _values;
|
||||
private readonly int _capacity;
|
||||
|
||||
// Number of full slots (observable count)
|
||||
private int _count;
|
||||
|
||||
// Next push position (0.._capacity). Only modified via CAS.
|
||||
private int _top;
|
||||
|
||||
public ConcurrentCapacityFixedStack(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
if (capacity <= 0) {
|
||||
throw new ArgumentOutOfRangeException(nameof(capacity));
|
||||
}
|
||||
|
||||
_capacity = capacity;
|
||||
_values = new T[capacity];
|
||||
_states = new int[capacity];
|
||||
_top = 0;
|
||||
_count = 0;
|
||||
}
|
||||
|
||||
public bool IsEmpty => Count == 0;
|
||||
|
||||
public int Capacity => _capacity;
|
||||
|
||||
public int Count => Volatile.Read(ref _count);
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
// Pop all elements to return slots to empty state
|
||||
while (TryPop(out _)) { }
|
||||
}
|
||||
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
T[] snap = ToArraySnapshot();
|
||||
for (int i = 0; i < snap.Length; i++) {
|
||||
yield return snap[i];
|
||||
}
|
||||
}
|
||||
|
||||
public T Peek()
|
||||
{
|
||||
if (TryPeek(out T? v)) {
|
||||
return v;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
if (TryPop(out T value)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
|
||||
public T[] PopRange(
|
||||
int count
|
||||
)
|
||||
{
|
||||
if (count <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
T[] result = new T[count];
|
||||
int i = 0;
|
||||
for (; i < count; i++)
|
||||
{
|
||||
if (!TryPop(out result[i]))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i == count) {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (i == 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
T[] trimmed = new T[i];
|
||||
Array.Copy(result, trimmed, i);
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
public void Push(
|
||||
T item
|
||||
)
|
||||
{
|
||||
SpinWait sw = default;
|
||||
while (true)
|
||||
{
|
||||
int oldTop = Volatile.Read(ref _top);
|
||||
if (oldTop >= _capacity) {
|
||||
throw new InvalidOperationException("Stack is full.");
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _top, oldTop + 1, oldTop) != oldTop)
|
||||
{
|
||||
sw.SpinOnce();
|
||||
continue; // failed to reserve slot
|
||||
}
|
||||
|
||||
// We exclusively own slot oldTop. Publish value then state.
|
||||
_values[oldTop] = item;
|
||||
Volatile.Write(ref _states[oldTop], 1); // publish
|
||||
_ = Interlocked.Increment(ref _count);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public void PushRange(
|
||||
ReadOnlySpan<T> span
|
||||
)
|
||||
{
|
||||
for (int i = 0; i < span.Length; i++) {
|
||||
Push(span[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort snapshot for enumeration; not strongly consistent under concurrent
|
||||
/// mutations.
|
||||
/// </summary>
|
||||
public T[] ToArraySnapshot()
|
||||
{
|
||||
int expected = Math.Min(Volatile.Read(ref _count), _capacity);
|
||||
if (expected == 0) {
|
||||
return Array.Empty<T>();
|
||||
}
|
||||
|
||||
T[] snap = new T[expected];
|
||||
int copied = 0;
|
||||
int idx = Volatile.Read(ref _top) - 1;
|
||||
while (idx >= 0 && copied < expected)
|
||||
{
|
||||
if (Volatile.Read(ref _states[idx]) == 1) {
|
||||
snap[copied++] = _values[idx];
|
||||
}
|
||||
idx--;
|
||||
}
|
||||
|
||||
if (copied == snap.Length) {
|
||||
return snap;
|
||||
}
|
||||
|
||||
T[] trimmed = new T[copied];
|
||||
Array.Copy(snap, trimmed, copied);
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
public bool TryPeek(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
value = default!;
|
||||
SpinWait sw = default;
|
||||
|
||||
while (true)
|
||||
{
|
||||
int top = Volatile.Read(ref _top);
|
||||
if (top <= 0) {
|
||||
return false;
|
||||
}
|
||||
int idx = top - 1;
|
||||
|
||||
// If not yet published or racing, retry/spin
|
||||
if (Volatile.Read(ref _states[idx]) == 0)
|
||||
{
|
||||
if (Volatile.Read(ref _top) != top)
|
||||
{
|
||||
continue; // head moved, retry
|
||||
}
|
||||
sw.SpinOnce();
|
||||
continue;
|
||||
}
|
||||
|
||||
T v = _values[idx];
|
||||
|
||||
// Validate consistency
|
||||
if (Volatile.Read(ref _top) == top && Volatile.Read(ref _states[idx]) == 1)
|
||||
{
|
||||
value = v;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryPop(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
value = default!;
|
||||
SpinWait sw = default;
|
||||
|
||||
while (true)
|
||||
{
|
||||
int oldTop = Volatile.Read(ref _top);
|
||||
if (oldTop <= 0)
|
||||
{
|
||||
return false; // empty
|
||||
}
|
||||
|
||||
int newTop = oldTop - 1; // index to consume
|
||||
if (Interlocked.CompareExchange(ref _top, newTop, oldTop) != oldTop)
|
||||
{
|
||||
sw.SpinOnce();
|
||||
continue; // failed to reserve element
|
||||
}
|
||||
|
||||
// If producer hasn't published yet, roll back our reservation and retry.
|
||||
if (Volatile.Read(ref _states[newTop]) == 0)
|
||||
{
|
||||
// attempt to restore top; ok if it fails due to concurrent operations
|
||||
_ = Interlocked.CompareExchange(ref _top, newTop + 1, newTop);
|
||||
sw.SpinOnce();
|
||||
continue;
|
||||
}
|
||||
|
||||
value = _values[newTop];
|
||||
_values[newTop] = default!; // avoid ref retention
|
||||
Volatile.Write(ref _states[newTop], 0); // mark empty
|
||||
_ = Interlocked.Decrement(ref _count);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public int TryPopRange(
|
||||
Span<T> destination
|
||||
)
|
||||
{
|
||||
if (destination.IsEmpty) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int i = 0;
|
||||
for (; i < destination.Length; i++)
|
||||
{
|
||||
if (!TryPop(out destination[i]))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ConcurrentFlexibleArray"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public class ConcurrentFlexibleArray<T>(
|
||||
int capacity = 4
|
||||
)
|
||||
{
|
||||
|
||||
private volatile T[] _array = new T[capacity];
|
||||
private readonly ReaderWriterLockSlim _rwLock = new();
|
||||
|
||||
public T this[
|
||||
int index
|
||||
]
|
||||
{
|
||||
get
|
||||
{
|
||||
_rwLock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
if ((uint)index >= (uint)_array.Length) {
|
||||
throw new ArgumentOutOfRangeException(nameof(index));
|
||||
}
|
||||
|
||||
return _array[index];
|
||||
}
|
||||
finally
|
||||
{
|
||||
_rwLock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
set
|
||||
{
|
||||
if ((uint)index >= (uint)_array.Length)
|
||||
{
|
||||
_rwLock.EnterWriteLock();
|
||||
try
|
||||
{
|
||||
if ((uint)index >= (uint)_array.Length)
|
||||
{
|
||||
int newSize = Math.Max(_array.Length * 2, index + 1);
|
||||
T[] newArray = new T[newSize];
|
||||
Array.Copy(_array, newArray, _array.Length);
|
||||
_array = newArray;
|
||||
_array[index] = value;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_rwLock.ExitWriteLock();
|
||||
}
|
||||
} else
|
||||
{
|
||||
_rwLock.EnterReadLock();
|
||||
try
|
||||
{
|
||||
_array[index] = value;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_rwLock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal Span<T> AsSpan()
|
||||
{
|
||||
return _array.AsSpan();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal void ReleaseSpanOperation()
|
||||
{
|
||||
_rwLock.ExitReadLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal void RequestSpanOperation()
|
||||
{
|
||||
_rwLock.EnterReadLock();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="IBidirectionalDictionary"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public interface IBidirectionalDictionary<TFirst, TSecond> : IDictionary<TFirst, TSecond>
|
||||
{
|
||||
|
||||
TFirst this[
|
||||
TSecond key
|
||||
] { get; set; }
|
||||
|
||||
ICollection<TFirst> FirstIndex { get; }
|
||||
|
||||
ICollection<TSecond> SecondIndex { get; }
|
||||
|
||||
IReadOnlyDictionary<TFirst, TSecond> Forward { get; }
|
||||
|
||||
IReadOnlyDictionary<TSecond, TFirst> Reverse { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains the
|
||||
/// specified key.
|
||||
/// </summary>
|
||||
/// <param _label="key">
|
||||
/// The value to locate in the <see cref="IBidirectionalDictionary{TFirst, TSecond}" />.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// true if the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains an element with
|
||||
/// the specified key; otherwise, false.
|
||||
/// </returns>
|
||||
bool Contains(
|
||||
TSecond key
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains the
|
||||
/// specified key.
|
||||
/// </summary>
|
||||
/// <param _label="key">
|
||||
/// The value to locate in the <see cref="IBidirectionalDictionary{TFirst, TSecond}" />.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// true if the <see cref="IBidirectionalDictionary{TFirst, TSecond}" /> contains an element with
|
||||
/// the specified key; otherwise, false.
|
||||
/// </returns>
|
||||
bool Contains(
|
||||
TFirst key
|
||||
);
|
||||
|
||||
bool TryGetValue(
|
||||
TSecond key,
|
||||
out TFirst value
|
||||
);
|
||||
|
||||
#pragma warning disable CA1033
|
||||
ICollection<TFirst> IDictionary<TFirst, TSecond>.Keys => FirstIndex;
|
||||
|
||||
ICollection<TSecond> IDictionary<TFirst, TSecond>.Values => SecondIndex;
|
||||
|
||||
bool IDictionary<TFirst, TSecond>.ContainsKey(
|
||||
TFirst key
|
||||
)
|
||||
{
|
||||
return Contains(key);
|
||||
}
|
||||
#pragma warning restore CA1033
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="LinearBag"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public static class LinearBagBuilder
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Build a <see cref="LinearBag{T}" /> from a read-only span. Capacity will exactly fit
|
||||
/// the input.
|
||||
/// </summary>
|
||||
public static LinearBag<T> Create<T>(
|
||||
ReadOnlySpan<T> values
|
||||
)
|
||||
{
|
||||
if (values.IsEmpty) {
|
||||
return new LinearBag<T>(4);
|
||||
}
|
||||
LinearBag<T> bag = new(values.Length);
|
||||
for (int i = 0; i < values.Length; i++) {
|
||||
bag.Add(values[i]);
|
||||
}
|
||||
return bag;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A very lightweight contiguous storage structure: unordered, not enumerable by design.
|
||||
/// Provides O(1) Add and O(1) RemoveAt (by swapping with the tail). Intended for high-
|
||||
/// performance scenarios where element ordering does not matter. NOT thread-safe.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// Element type.
|
||||
/// </typeparam>
|
||||
[CollectionBuilder(typeof(LinearBagBuilder), nameof(LinearBagBuilder.Create))]
|
||||
public class LinearBag<T>(
|
||||
int capacity = 4
|
||||
)
|
||||
{
|
||||
|
||||
// Invariant: valid elements occupy indexes [0, Count-1]. Count is the number of live elements.
|
||||
private T[] _items = new T[Math.Max(0, capacity)];
|
||||
|
||||
/// <summary>
|
||||
/// Access element by index (0..Count-1). Throws if out of range.
|
||||
/// </summary>
|
||||
public T this[
|
||||
int index
|
||||
]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count);
|
||||
return _items[index];
|
||||
}
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
set
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(index, Count);
|
||||
_items[index] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Total allocated slots.
|
||||
/// </summary>
|
||||
public int Capacity => _items.Length;
|
||||
|
||||
/// <summary>
|
||||
/// Number of currently stored elements (valid index range 0..Count-1).
|
||||
/// </summary>
|
||||
public int Count { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Adds an item (unordered). Expands internal array if needed.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Add(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (Count == _items.Length) {
|
||||
EnsureCapacity(Count + 1);
|
||||
}
|
||||
|
||||
_items[Count] = item;
|
||||
Count++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a span over the live portion of the bag (0..Count-1).
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Span<T> AsSpan()
|
||||
{
|
||||
return new Span<T>(_items, 0, Count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Clears all elements and zeroes out references to avoid retention. Count becomes 0.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Clear()
|
||||
{
|
||||
Array.Clear(_items, 0, Count);
|
||||
Count = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fast clear: only resets Count (leaves underlying array unchanged). References are NOT
|
||||
/// released. Use only when you are certain old references can remain.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void FastClear()
|
||||
{
|
||||
Count = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the first occurrence of value (unordered). Does nothing if not found.
|
||||
/// </summary>
|
||||
public void Remove(
|
||||
T value,
|
||||
EqualityComparer<T> comparer = default
|
||||
)
|
||||
{
|
||||
comparer ??= EqualityComparer<T>.Default;
|
||||
for (int i = 0; i < Count; i++)
|
||||
{
|
||||
if (comparer.Equals(_items[i], value))
|
||||
{
|
||||
RemoveAt(i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes element at index by swapping the last element into its slot (unless already
|
||||
/// last). Preserves O(1) removal at cost of ordering.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void RemoveAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
if ((uint)index >= (uint)Count) {
|
||||
throw new ArgumentOutOfRangeException(nameof(index));
|
||||
}
|
||||
|
||||
int last = Count - 1;
|
||||
if (index != last) {
|
||||
_items[index] = _items[last];
|
||||
}
|
||||
|
||||
_items[last] = default!; // release reference / reset value
|
||||
Count = last;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures capacity is at least <paramref name="min" />. Doubles size when growing.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void EnsureCapacity(
|
||||
int min
|
||||
)
|
||||
{
|
||||
if (_items.Length >= min) {
|
||||
return;
|
||||
}
|
||||
|
||||
int newCapacity = (_items.Length == 0) ? 4 : (_items.Length * 2);
|
||||
if (newCapacity < min) {
|
||||
newCapacity = min;
|
||||
}
|
||||
|
||||
Array.Resize(ref _items, newCapacity);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ListSegment"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Serialization;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Part of a collection. This contains a direct reference to _origin collection, so all
|
||||
/// change to _origin collection will be reflected in this object.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
///
|
||||
/// </typeparam>
|
||||
public struct ListSegment<T> : ICollection<T>
|
||||
{
|
||||
|
||||
private int _begin;
|
||||
private int _length;
|
||||
private List<T> _collection;
|
||||
|
||||
public ListSegment(
|
||||
List<T> collection,
|
||||
int begin,
|
||||
int length
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(collection);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(begin, 0);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(length, 0);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(begin + length, collection.Count);
|
||||
_collection = collection;
|
||||
_begin = begin;
|
||||
_length = length;
|
||||
}
|
||||
|
||||
public T this[
|
||||
int indexValue
|
||||
]
|
||||
{
|
||||
get
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(indexValue, 0);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(indexValue, _length);
|
||||
return _collection[indexValue + _begin];
|
||||
}
|
||||
}
|
||||
|
||||
public T this[
|
||||
Index i
|
||||
]
|
||||
{
|
||||
get
|
||||
{
|
||||
int offset = i.IsFromEnd ? (_length - i.Value) : i.Value;
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(offset, 0);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, _length);
|
||||
return _collection[offset + _begin];
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsReadOnly => true;
|
||||
|
||||
public int Count => _length;
|
||||
|
||||
public bool Contains(
|
||||
T item
|
||||
)
|
||||
{
|
||||
int index = _collection.IndexOf(item);
|
||||
return (index >= _begin) && (index < _length + index);
|
||||
}
|
||||
|
||||
public void CopyTo(
|
||||
T[] array,
|
||||
int arrayIndex
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(array);
|
||||
Span<T> span = CollectionsMarshal.AsSpan<T>(_collection).Slice(_begin, _length);
|
||||
Span<T> target = new(array, arrayIndex, _length);
|
||||
span.CopyTo(target);
|
||||
}
|
||||
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
for (int i = _begin; i < _length; i++) {
|
||||
yield return this[i];
|
||||
}
|
||||
}
|
||||
|
||||
void ICollection<T>.Add(
|
||||
T item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
void ICollection<T>.Clear()
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
bool ICollection<T>.Remove(
|
||||
T item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="RandomAccessibleStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public class RandomAccessibleStack<T> : IEnumerable<T>
|
||||
{
|
||||
|
||||
private const int DefaultCapacity = 4;
|
||||
|
||||
private T[] _array;
|
||||
private int _size;
|
||||
private int _version;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the RandomAccessStack class that is empty and has the
|
||||
/// default initial capacity.
|
||||
/// </summary>
|
||||
public RandomAccessibleStack()
|
||||
{
|
||||
_array = new T[DefaultCapacity];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the RandomAccessStack class that is empty and has the
|
||||
/// specified initial capacity.
|
||||
/// </summary>
|
||||
public RandomAccessibleStack(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
if (capacity < 0) {
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(capacity), "Capacity must be non-negative.");
|
||||
}
|
||||
|
||||
_array = new T[capacity];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the element at the specified index in the stack (0 = bottom, Count-1 =
|
||||
/// top).
|
||||
/// </summary>
|
||||
public T this[
|
||||
int index
|
||||
]
|
||||
{
|
||||
get
|
||||
{
|
||||
if (index < 0 || index >= _size) {
|
||||
throw new ArgumentOutOfRangeException(nameof(index));
|
||||
}
|
||||
|
||||
return _array[index];
|
||||
}
|
||||
/*
|
||||
set
|
||||
{
|
||||
if( index < 0 || index >= PrimSize ) {
|
||||
throw new ArgumentOutOfRangeException( nameof( index ) );
|
||||
}
|
||||
_array[ index ] = value;
|
||||
_version++;
|
||||
}
|
||||
/**/
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the number of elements contained in the stack.
|
||||
/// </summary>
|
||||
public int Count => _size;
|
||||
|
||||
/// <summary>
|
||||
/// Removes all objects from the stack.
|
||||
/// </summary>
|
||||
public void Clear()
|
||||
{
|
||||
Array.Clear(_array, 0, _size);
|
||||
_size = 0;
|
||||
_version++;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns an enumerator that iterates through the stack from top to bottom. Throws if
|
||||
/// the stack is modified during iteration.
|
||||
/// </summary>
|
||||
public IEnumerator<T> GetEnumerator()
|
||||
{
|
||||
int expectedVersion = _version;
|
||||
for (int i = _size - 1; i >= 0; i--)
|
||||
{
|
||||
if (expectedVersion != _version) {
|
||||
throw new InvalidOperationException(
|
||||
"Collection was modified during enumeration.");
|
||||
}
|
||||
|
||||
yield return _array[i];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the object at the top of the stack without removing it.
|
||||
/// </summary>
|
||||
public T Peek()
|
||||
{
|
||||
if (_size == 0) {
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
|
||||
return _array[_size - 1];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes and returns the object at the top of the stack.
|
||||
/// </summary>
|
||||
public T Pop()
|
||||
{
|
||||
if (_size == 0) {
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
|
||||
T item = _array[--_size];
|
||||
_array[_size] = default!;
|
||||
_version++;
|
||||
return item;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an item to the top of the stack.
|
||||
/// </summary>
|
||||
public void Push(
|
||||
T item
|
||||
)
|
||||
{
|
||||
if (_size == _array.Length) {
|
||||
EnsureCapacity(_size + 1);
|
||||
}
|
||||
|
||||
_array[_size++] = item;
|
||||
_version++;
|
||||
}
|
||||
|
||||
private void EnsureCapacity(
|
||||
int min
|
||||
)
|
||||
{
|
||||
int newCapacity = _array.Length == 0 ? DefaultCapacity : _array.Length * 2;
|
||||
if (newCapacity < min) {
|
||||
newCapacity = min;
|
||||
}
|
||||
|
||||
Array.Resize(ref _array, newCapacity);
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissTable.X86"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public partial class SwissTable
|
||||
{
|
||||
|
||||
private unsafe class BucketScannerX86 : IBucketScanner
|
||||
{
|
||||
|
||||
public bool TryFind(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
Vector128<byte> bucketCtrlVector = Sse2.LoadVector128((byte*)&bucket->Control);
|
||||
Vector128<byte> cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector);
|
||||
int mask = Sse2.MoveMask(cmpResult);
|
||||
while (mask != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(mask);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
return true;
|
||||
}
|
||||
mask &= mask - 1;
|
||||
}
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool TryRemove(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F);
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
|
||||
Vector128<byte> bucketCtrl = Sse2.LoadVector128((byte*)&bucket->Control);
|
||||
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
|
||||
int matches = Sse2.MoveMask(matchMask);
|
||||
|
||||
while (matches != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(matches);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
value = bucket->Values[index];
|
||||
bucket->Control[index] = 0xFE; // mark as DELETED
|
||||
// no need to remove Key/Value
|
||||
return true;
|
||||
}
|
||||
matches &= matches - 1;
|
||||
}
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
public bool TrySet(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
)
|
||||
{
|
||||
byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0)
|
||||
Vector128<byte> ctrlVector = Vector128.Create(ctrl);
|
||||
|
||||
Vector128<byte> bucketCtrl = Sse2.LoadVector128(
|
||||
(byte*)&bucket->Control);
|
||||
|
||||
// === Phase 1: Look up an existing key ===
|
||||
Vector128<byte> matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl);
|
||||
int matches = Sse2.MoveMask(matchMask);
|
||||
|
||||
while (matches != 0)
|
||||
{
|
||||
int bitPos = BitOperations.TrailingZeroCount(matches);
|
||||
int index = bitPos;
|
||||
if (bucket->Keys[index] == key)
|
||||
{
|
||||
// Key already exists -> update value
|
||||
bucket->Values[index] = value;
|
||||
isNewKey = false;
|
||||
return true;
|
||||
}
|
||||
matches &= matches - 1;
|
||||
}
|
||||
|
||||
// === Phase 2: Track an available insertion slot ===
|
||||
int emptyOrDeleted = Sse2.MoveMask(bucketCtrl);
|
||||
if (emptyOrDeleted == 0)
|
||||
{
|
||||
isNewKey = false;
|
||||
return false; // bucket full
|
||||
}
|
||||
|
||||
int slot = BitOperations.TrailingZeroCount(emptyOrDeleted);
|
||||
bucket->Control[slot] = ctrl;
|
||||
bucket->Keys[slot] = key;
|
||||
bucket->Values[slot] = value;
|
||||
isNewKey = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SwissTable"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics.Arm;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
public sealed unsafe partial class SwissTable : IDisposable
|
||||
{
|
||||
|
||||
private const int BucketCount = 1 << EntryMaskWidth;
|
||||
private const int BucketSize = 16;
|
||||
private const uint CrcMagic = 0xffff9961; // Magic number for CRC32C, used in hash computation
|
||||
private const int EntryMaskWidth = 10; // 1024 entries total, 16 per bucket
|
||||
public const int MaxCapacity = BucketCount * BucketSize; // 256 entries per bucket, 1024 buckets
|
||||
|
||||
public static readonly nuint RequestedNativeMemorySize = (nuint)Unsafe.SizeOf<Entry>();
|
||||
|
||||
private readonly Entry* _entries;
|
||||
private readonly Action<nuint> _freeHandle;
|
||||
private bool _disposed;
|
||||
|
||||
private SwissTable(
|
||||
Entry* buffer,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
_entries = buffer;
|
||||
_freeHandle = freeHandle;
|
||||
}
|
||||
|
||||
public static SwissTable Create()
|
||||
{
|
||||
Entry* buffer = (Entry*)NativeMemory.AlignedAlloc((nuint)Unsafe.SizeOf<Entry>(), 16);
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, handle => NativeMemory.AlignedFree((void*)handle));
|
||||
}
|
||||
|
||||
public static SwissTable Create(
|
||||
Func<nuint, nuint> allocator,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
Entry* buffer = (Entry*)allocator(RequestedNativeMemorySize);
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, freeHandle);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_freeHandle((nuint)_entries);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public bool TryAddOrSet(
|
||||
nuint key,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, (ulong)key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey);
|
||||
}
|
||||
|
||||
public bool TryGet(
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TryFind(key, hash, bucket, out value);
|
||||
/*
|
||||
* hash = CRC32 of key
|
||||
* entry = low 8 of hash
|
||||
* ctrl = 8-15 of hash
|
||||
* index = rest of hash
|
||||
* find value of hash
|
||||
*/
|
||||
}
|
||||
|
||||
public bool TryRemove(
|
||||
nuint key,
|
||||
out nuint value
|
||||
)
|
||||
{
|
||||
uint hash = BitOperations.Crc32C(CrcMagic, key);
|
||||
ushort entry = (ushort)(hash & (BucketCount - 1));
|
||||
Bucket* bucket = &(*_entries)[entry];
|
||||
return IBucketScanner.Current.TryRemove(key, hash, bucket, out value);
|
||||
}
|
||||
|
||||
internal static SwissTable Create(
|
||||
nuint bufferPtr,
|
||||
Action<nuint> FreeHandle
|
||||
)
|
||||
{
|
||||
Entry* buffer = (Entry*)bufferPtr;
|
||||
for (int i = 0; i < BucketCount; i++)
|
||||
{
|
||||
// Set all control bytes to 0xFF (empty)
|
||||
Unsafe.InitBlock(&(*buffer)[i].Control, 0xFF, BucketSize);
|
||||
}
|
||||
return new SwissTable(buffer, FreeHandle);
|
||||
}
|
||||
|
||||
~SwissTable()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
[InlineArray(BucketCount)]
|
||||
private struct Entry
|
||||
{
|
||||
|
||||
public Bucket Bucket;
|
||||
|
||||
}
|
||||
|
||||
private unsafe struct Bucket
|
||||
{
|
||||
|
||||
public BucketControl Control;
|
||||
public BucketKeyStorage Keys;
|
||||
public BucketValueStorage Values;
|
||||
|
||||
}
|
||||
|
||||
private interface IBucketScanner
|
||||
{
|
||||
|
||||
static IBucketScanner Current { get; } = Create();
|
||||
|
||||
bool TryFind(
|
||||
nuint key,
|
||||
uint Hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
);
|
||||
|
||||
unsafe bool TryRemove(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
out nuint value
|
||||
);
|
||||
|
||||
bool TrySet(
|
||||
nuint key,
|
||||
uint hash,
|
||||
Bucket* bucket,
|
||||
nuint value,
|
||||
out bool isNewKey
|
||||
);
|
||||
|
||||
private static IBucketScanner Create()
|
||||
{
|
||||
if (Sse42.IsSupported)
|
||||
{
|
||||
return new BucketScannerX86();
|
||||
} else if (ArmBase.IsSupported)
|
||||
{
|
||||
throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform.");
|
||||
} else
|
||||
{
|
||||
throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketControl
|
||||
{
|
||||
|
||||
public byte Control;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketKeyStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
[InlineArray(BucketSize)]
|
||||
private struct BucketValueStorage
|
||||
{
|
||||
|
||||
public nuint value;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="UnmanagedArray"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
/// <summary>
|
||||
/// Zero-overhead view over a contiguous region of unmanaged memory. Does not own the memory —
|
||||
/// caller is responsible for lifetime management.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// Element type. Must be unmanaged.
|
||||
/// </typeparam>
|
||||
public readonly unsafe struct UnmanagedArray<T> where T : unmanaged
|
||||
{
|
||||
|
||||
private static readonly nint ElementSize = sizeof(T);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public UnmanagedArray(
|
||||
T* pointer,
|
||||
int length
|
||||
)
|
||||
{
|
||||
Address = pointer;
|
||||
Length = length;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public UnmanagedArray(
|
||||
nuint address,
|
||||
int length
|
||||
)
|
||||
{
|
||||
Address = (T*)address;
|
||||
Length = length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Indexed access. No bounds check in Release builds.
|
||||
/// </summary>
|
||||
public ref T this[
|
||||
int index
|
||||
]
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get
|
||||
{
|
||||
Debug.Assert((uint)index < (uint)Length,
|
||||
"UnmanagedArray index out of range");
|
||||
return ref Address[index];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Raw pointer to the first element.
|
||||
/// </summary>
|
||||
public T* Address
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get;
|
||||
init;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Element count.
|
||||
/// </summary>
|
||||
public int Length
|
||||
{
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
get;
|
||||
init;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Total byte size of the array (for allocation calculations).
|
||||
/// </summary>
|
||||
public int ByteSize => Length * sizeof(T);
|
||||
|
||||
/// <summary>
|
||||
/// Wraps the array as a <see cref="Span{T}" />.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Span<T> AsSpan()
|
||||
{
|
||||
return new Span<T>(Address, Length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wraps a sub-range as a <see cref="Span{T}" />.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Span<T> AsSpan(
|
||||
int start,
|
||||
int length
|
||||
)
|
||||
{
|
||||
Debug.Assert(((uint)start) + ((uint)length) <= (uint)Length,
|
||||
"UnmanagedArray.AsSpan range out of bounds");
|
||||
return new Span<T>(Address + start, length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Zero-fill all elements.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Clear()
|
||||
{
|
||||
Unsafe.InitBlock(Address, 0, (uint)(Length * sizeof(T)));
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int IndexOf(
|
||||
T* ptr
|
||||
)
|
||||
{
|
||||
int offset = (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(offset, nameof(ptr));
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, Length, nameof(ptr));
|
||||
return offset;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a pointer to element at <paramref name="index" />. Useful for feeding SIMD
|
||||
/// load instructions (e.g. Sse2.LoadVector128).
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T* PtrAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
Debug.Assert((uint)index < (uint)Length,
|
||||
"UnmanagedArray.PtrAt index out of range");
|
||||
return Address + index;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int UncheckedIndexOf(
|
||||
T* ptr
|
||||
)
|
||||
{
|
||||
return (int)((((nint)ptr) - ((nint)Address)) / ElementSize);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="UnmanagedStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Collections
|
||||
{
|
||||
public unsafe class UnmanagedStack<T>(
|
||||
T* address,
|
||||
int capacity,
|
||||
Action<nuint> freeHandle
|
||||
) : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
private T* _array = address;
|
||||
private readonly Action<nuint> _freeHandle = freeHandle;
|
||||
private readonly int _capacity = capacity;
|
||||
private int _index;
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
_index = 0;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_array != null)
|
||||
{
|
||||
_freeHandle((nuint)_array);
|
||||
_array = null;
|
||||
}
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
if (_index == 0) {
|
||||
throw new InvalidOperationException("Stack is empty.");
|
||||
}
|
||||
return _array[--_index];
|
||||
}
|
||||
|
||||
public void Push(
|
||||
T value
|
||||
)
|
||||
{
|
||||
_array[_index] = value;
|
||||
_index++;
|
||||
}
|
||||
|
||||
public bool TryPop(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
if (_index == 0)
|
||||
{
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
value = _array[--_index];
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool TryPush(
|
||||
T value
|
||||
)
|
||||
{
|
||||
if (_index >= _capacity) {
|
||||
return false;
|
||||
}
|
||||
_array[_index] = value;
|
||||
_index++;
|
||||
return true;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Dialogue"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.IO.Pipes;
|
||||
using System.Reflection;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
public static class Dialogue
|
||||
{
|
||||
|
||||
private static Process? _dialogueProcess;
|
||||
private static AnonymousPipeServerStream _dialogueClientPipe,_dialogueServerPipe;
|
||||
private static int _readAndWriteThreadId;
|
||||
private static ProcessStartInfo _dialogueProcessInfo;
|
||||
private static StreamReader _dr;
|
||||
private static StreamWriter _dw;
|
||||
|
||||
public static bool NeedShowWhenStartup { get; set; }
|
||||
|
||||
public static void PostError(
|
||||
Exception ex
|
||||
)
|
||||
{
|
||||
CreateDialogueProcessIfNotExist();
|
||||
|
||||
_dw.WriteLine($"posterror {ex.Message}");
|
||||
|
||||
_dialogueProcess!.Exited -= DialogueProcessExitCallback;
|
||||
}
|
||||
|
||||
public static void SetReadAndWriteThread(
|
||||
Thread thread
|
||||
)
|
||||
{
|
||||
_readAndWriteThreadId = thread.ManagedThreadId;
|
||||
}
|
||||
|
||||
internal static void CreateDialogueProcessIfNotExist()
|
||||
{
|
||||
if (_dialogueProcess != null) {
|
||||
return;
|
||||
}
|
||||
if (!File.Exists(_dialogueProcessInfo.FileName)) {
|
||||
return;
|
||||
}
|
||||
_dialogueProcess = Process.Start(_dialogueProcessInfo);
|
||||
|
||||
_dr = new StreamReader(_dialogueClientPipe);
|
||||
_dw = new StreamWriter(_dialogueServerPipe);
|
||||
|
||||
_dialogueProcess!.Exited += DialogueProcessExitCallback;
|
||||
|
||||
_dialogueServerPipe.DisposeLocalCopyOfClientHandle();
|
||||
_dialogueClientPipe.DisposeLocalCopyOfClientHandle();
|
||||
}
|
||||
|
||||
internal static void LoadDialogue()
|
||||
{
|
||||
_dialogueServerPipe = new AnonymousPipeServerStream(
|
||||
PipeDirection.Out, HandleInheritability.Inheritable);
|
||||
_dialogueClientPipe = new AnonymousPipeServerStream(
|
||||
PipeDirection.In, HandleInheritability.Inheritable);
|
||||
|
||||
string route = Assembly.GetAssembly(typeof(World))?.Location ?? string.Empty;
|
||||
string path = Path.GetDirectoryName(route) ?? string.Empty;
|
||||
|
||||
string exeName;
|
||||
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
||||
{
|
||||
exeName = ".exe";
|
||||
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
|
||||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
|
||||
{
|
||||
exeName = string.Empty;
|
||||
} else
|
||||
{
|
||||
throw new PlatformNotSupportedException("Unknown OS platform");
|
||||
}
|
||||
|
||||
string consoleRoute = Path.Combine(
|
||||
path, @"StgSharp.TerminalDialogue", $@"StgSharp.TerminalDialogue{exeName}");
|
||||
|
||||
_dialogueProcessInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = consoleRoute,
|
||||
Arguments =
|
||||
$"{_dialogueServerPipe.GetClientHandleAsString()} {_dialogueClientPipe.GetClientHandleAsString()}",
|
||||
UseShellExecute = true,
|
||||
RedirectStandardOutput = false,
|
||||
};
|
||||
}
|
||||
|
||||
internal static void PostMessage(
|
||||
string instruction,
|
||||
params string[] operands
|
||||
)
|
||||
{
|
||||
CreateDialogueProcessIfNotExist();
|
||||
|
||||
_dw.WriteLine($"{instruction} {operands}");
|
||||
}
|
||||
|
||||
internal static void UnloadDialogue()
|
||||
{
|
||||
if (_dialogueProcess != null) {
|
||||
_dialogueProcess.Kill();
|
||||
}
|
||||
_dialogueProcess = null;
|
||||
|
||||
_dialogueServerPipe.Dispose();
|
||||
_dialogueClientPipe.Dispose();
|
||||
}
|
||||
|
||||
internal static string WaitCertainReturn(
|
||||
string target
|
||||
)
|
||||
{
|
||||
if (_dialogueProcess == null) {
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
string temp = string .Empty;
|
||||
do
|
||||
{
|
||||
temp = _dr.ReadLine();
|
||||
temp = temp ?? string.Empty;
|
||||
if (!temp!.StartsWith(target))
|
||||
{
|
||||
break;
|
||||
}
|
||||
} while (true);
|
||||
|
||||
int index = temp.IndexOf(' ');
|
||||
return index != -1 ? temp.Substring(index + 1) : string.Empty;
|
||||
}
|
||||
|
||||
private static void AssertThread()
|
||||
{
|
||||
if (_readAndWriteThreadId != Environment.CurrentManagedThreadId) {
|
||||
throw new InvalidOperationException(
|
||||
"Communication with dialogue can only call from certain thread.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void DialogueProcessExitCallback(
|
||||
object? sender,
|
||||
EventArgs e
|
||||
)
|
||||
{
|
||||
_dialogueProcess = null;
|
||||
_dr.Close();
|
||||
_dw.Close();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ErrorDefine"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
public class UniformEmptyReferenceException : Exception
|
||||
{
|
||||
|
||||
public UniformEmptyReferenceException(
|
||||
string program,
|
||||
string name
|
||||
)
|
||||
: base(
|
||||
$"Location -1 returned when trying to find uniform {name} in shader program {program}") { }
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="GlobalSetting"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
public static unsafe partial class World
|
||||
{
|
||||
|
||||
internal const int ssdSegmentLength = 16;
|
||||
|
||||
private static byte[] _currentHash = [];
|
||||
|
||||
private static bool _inited;
|
||||
|
||||
public static byte[] CurrentAssemblyHash
|
||||
{
|
||||
get
|
||||
{
|
||||
if (_currentHash == null)
|
||||
{
|
||||
string route = Assembly.GetExecutingAssembly().Location;
|
||||
string time = DateTime.UtcNow.ToString();
|
||||
using (SHA256 sh = SHA256.Create())
|
||||
{
|
||||
byte[] bytes = Encoding.UTF8.GetBytes(route);
|
||||
_currentHash = sh.ComputeHash(bytes);
|
||||
}
|
||||
}
|
||||
return _currentHash;
|
||||
}
|
||||
}
|
||||
|
||||
public static int MainThreadID
|
||||
{
|
||||
get => (_mainThreadID == -1) ?
|
||||
throw new InvalidOperationException(
|
||||
"StgSharp environment is not inited. Main thread id is not available") :
|
||||
_mainThreadID;
|
||||
internal set => _mainThreadID = value;
|
||||
}
|
||||
|
||||
private static int _mainThreadID { get; set; } = -1;
|
||||
|
||||
public static class GlobalSetting
|
||||
{
|
||||
|
||||
private static bool vSyncActivated;
|
||||
|
||||
public static bool VSyncActivated
|
||||
{
|
||||
get => vsyncActivated;
|
||||
set =>
|
||||
// GraphicFramework.glfwSwapInterval(value ? 1 : 0);
|
||||
vsyncActivated = value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}//-------------------------------------- End of Class ---------------------------------------//
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="IStaticModule"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a statically activated module that can be registered and initialized as part of
|
||||
/// the application startup sequence, and gracefully torn down on shutdown.
|
||||
/// </summary>
|
||||
public interface IStaticModule
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Gets the display name of this module, used for logging and diagnostics.
|
||||
/// </summary>
|
||||
string ModuleName { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Performs all initialization work for this module using the provided profile.
|
||||
/// </summary>
|
||||
/// <param name="profile">
|
||||
/// The configuration profile supplied at registration time. Use <see
|
||||
/// cref="IModuleInitializeProfile.Empty" /> if no configuration is required.
|
||||
/// </param>
|
||||
void InitializeModule(
|
||||
IModuleInitializeProfile profile
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Releases all resources acquired during <see cref="InitializeModule" />. Implementations should
|
||||
/// be idempotent: calling this method more than once must not throw.
|
||||
/// </summary>
|
||||
void UninitializeModule();
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Marker interface for module initialization configuration. Implementations are plain data
|
||||
/// carriers; no static factory is required.
|
||||
/// </summary>
|
||||
public interface IModuleInitializeProfile
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// A shared, allocation-free empty profile for modules that require no configuration.
|
||||
/// </summary>
|
||||
static readonly IModuleInitializeProfile Empty = EmptyModuleInitializeProfile.Instance;
|
||||
|
||||
private sealed class EmptyModuleInitializeProfile : IModuleInitializeProfile
|
||||
{
|
||||
|
||||
public static readonly EmptyModuleInitializeProfile Instance = new();
|
||||
|
||||
private EmptyModuleInitializeProfile() { }
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pairs an <see cref="IStaticModule" /> with its <see cref="IModuleInitializeProfile" />,
|
||||
/// ensuring the two are never stored or iterated independently.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Using a <see langword="readonly struct" /> avoids the boxing overhead that would occur if
|
||||
/// the pair were stored as two parallel lists of reference types.
|
||||
/// </remarks>
|
||||
public readonly struct ModuleDescriptor
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="ModuleDescriptor" /> using <see
|
||||
/// cref="IModuleInitializeProfile.Empty" /> as the profile.
|
||||
/// </summary>
|
||||
public ModuleDescriptor(
|
||||
IStaticModule module
|
||||
)
|
||||
: this(module, IModuleInitializeProfile.Empty) { }
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="ModuleDescriptor" /> with an explicit profile.
|
||||
/// </summary>
|
||||
public ModuleDescriptor(
|
||||
IStaticModule module,
|
||||
IModuleInitializeProfile profile
|
||||
)
|
||||
{
|
||||
Module = module;
|
||||
Profile = profile;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the configuration profile associated with the module.
|
||||
/// </summary>
|
||||
public IModuleInitializeProfile Profile { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the module to be initialized.
|
||||
/// </summary>
|
||||
public IStaticModule Module { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Calls <see cref="IStaticModule.InitializeModule" /> with the stored profile.
|
||||
/// </summary>
|
||||
public void Initialize()
|
||||
{
|
||||
Module.InitializeModule(Profile);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calls <see cref="IStaticModule.UninitializeModule" /> on the stored module.
|
||||
/// </summary>
|
||||
public void Uninitialize()
|
||||
{
|
||||
Module.UninitializeModule();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An ordered collection of modules to be initialized during application startup and
|
||||
/// uninitialized on shutdown. Supports three registration strategies so that the <c> new()
|
||||
/// </c> generic constraint is never a hard requirement.
|
||||
/// </summary>
|
||||
public sealed class ModuleToInitializeCollection : IEnumerable<IStaticModule>, IStaticModule
|
||||
{
|
||||
|
||||
private readonly List<ModuleDescriptor> _descriptors = [];
|
||||
|
||||
internal ModuleToInitializeCollection(
|
||||
World.Initializer i
|
||||
)
|
||||
{
|
||||
_descriptors.Add(new ModuleDescriptor(i));
|
||||
}
|
||||
|
||||
// ── IEnumerable<IStaticModule> ───────────────────────────────────────────
|
||||
|
||||
/// <inheritdoc />
|
||||
public IEnumerator<IStaticModule> GetEnumerator()
|
||||
{
|
||||
foreach (ModuleDescriptor d in _descriptors) {
|
||||
yield return d.Module;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Initialization / Uninitialization ────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Initializes all registered modules in the order they were registered. Each module
|
||||
/// receives the profile that was supplied at registration time.
|
||||
/// </summary>
|
||||
public void InitializeModule()
|
||||
{
|
||||
for (int i = 0; i < _descriptors.Count; i++) {
|
||||
_descriptors[i].Initialize();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uninitializes all registered modules in <b> reverse </b> registration order, mirroring
|
||||
/// the typical LIFO teardown convention (e.g., GLFW must be unloaded after all subsystems
|
||||
/// that depend on it).
|
||||
/// </summary>
|
||||
public void UninitializeModule()
|
||||
{
|
||||
for (int i = _descriptors.Count - 1; i >= 0; i--) {
|
||||
_descriptors[i].Uninitialize();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Registration API ─────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Registers a module of type <typeparamref name="T" /> using its default constructor.
|
||||
/// This is the most concise overload for simple modules that expose a public
|
||||
/// parameterless constructor.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// A concrete module type that implements <see cref="IStaticModule" /> and exposes a
|
||||
/// public parameterless constructor.
|
||||
/// </typeparam>
|
||||
/// <param name="profile">
|
||||
/// Optional initialization profile. Defaults to <see
|
||||
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
|
||||
/// </param>
|
||||
public ModuleToInitializeCollection UseModule<T>(
|
||||
IModuleInitializeProfile? profile = null
|
||||
) where T : IStaticModule, new()
|
||||
{
|
||||
return UseModule(static() => new T(), profile);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a module produced by a factory delegate. This overload imposes no generic
|
||||
/// constraints and supports any construction strategy, including dependency injection and
|
||||
/// parameterized constructors.
|
||||
/// </summary>
|
||||
/// <param name="factory">
|
||||
/// A delegate that constructs and returns the module instance. The delegate is invoked
|
||||
/// once, immediately at registration time. If <paramref name="factory" /> is <see
|
||||
/// langword="null" /> or returns <see langword="null" />, the call is silently ignored
|
||||
/// and the collection is returned unchanged.
|
||||
/// </param>
|
||||
/// <param name="profile">
|
||||
/// Optional initialization profile. Defaults to <see
|
||||
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
|
||||
/// </param>
|
||||
public ModuleToInitializeCollection UseModule(
|
||||
Func<IStaticModule> factory,
|
||||
IModuleInitializeProfile? profile = null
|
||||
)
|
||||
{
|
||||
if (factory is null) {
|
||||
return this;
|
||||
}
|
||||
IStaticModule? mod = factory();
|
||||
if (mod is null) {
|
||||
return this;
|
||||
}
|
||||
_descriptors.Add(new ModuleDescriptor(
|
||||
mod,
|
||||
profile ?? IModuleInitializeProfile.Empty));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers a pre-existing module instance. Use this overload when the module is
|
||||
/// provided externally, such as from a DI container, an object pool, or a singleton.
|
||||
/// </summary>
|
||||
/// <param name="module">
|
||||
/// The module instance to register.
|
||||
/// </param>
|
||||
/// <param name="profile">
|
||||
/// Optional initialization profile. Defaults to <see
|
||||
/// cref="IModuleInitializeProfile.Empty" /> when <see langword="null" />.
|
||||
/// </param>
|
||||
public ModuleToInitializeCollection UseModule(
|
||||
IStaticModule module,
|
||||
IModuleInitializeProfile? profile = null
|
||||
)
|
||||
{
|
||||
_descriptors.Add(new ModuleDescriptor(
|
||||
module,
|
||||
profile ?? IModuleInitializeProfile.Empty));
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
// ── IStaticModule explicit implementation ────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Explicit <see cref="IStaticModule" /> implementation. Delegates to <see
|
||||
/// cref="InitializeModule()" />, ignoring the supplied profile because each contained module uses
|
||||
/// its own registered profile.
|
||||
/// </summary>
|
||||
void IStaticModule.InitializeModule(
|
||||
IModuleInitializeProfile profile
|
||||
)
|
||||
{
|
||||
InitializeModule();
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
string IStaticModule.ModuleName => "Module Initializer";
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Platform"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
internal enum GraphicAPI : byte
|
||||
{
|
||||
|
||||
GL,
|
||||
VK
|
||||
|
||||
}
|
||||
|
||||
public static partial class World
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Version of current World platform. Main version represent in hundred's digit, and sub
|
||||
/// version represent in ten's digit.
|
||||
/// </summary>
|
||||
public const long version = 430;
|
||||
|
||||
private static bool vsyncActivated;
|
||||
|
||||
private static uint markCount = 0;
|
||||
internal static GraphicAPI API = default;
|
||||
|
||||
public static ModuleToInitializeCollection Initialize()
|
||||
{
|
||||
return new ModuleToInitializeCollection(new Initializer());
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void LogError(
|
||||
Exception ex
|
||||
)
|
||||
{
|
||||
DefaultLog.InternalWriteLog(ex.Message, LogType.Error);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void LogInfo(
|
||||
string log
|
||||
)
|
||||
{
|
||||
DefaultLog.InternalWriteLog(log, LogType.Info);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void LogWarning(
|
||||
string log
|
||||
)
|
||||
{
|
||||
DefaultLog.InternalWriteLog(log, LogType.Warning);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void LogWarning(
|
||||
Exception notVerySeriesException
|
||||
)
|
||||
{
|
||||
DefaultLog.InternalWriteLog(notVerySeriesException.Message, LogType.Warning);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void Mark(
|
||||
bool needConsole
|
||||
)
|
||||
{
|
||||
markCount++;
|
||||
if (needConsole) {
|
||||
Console.WriteLine(markCount);
|
||||
}
|
||||
}
|
||||
|
||||
public static void Terminate(
|
||||
int exitCode
|
||||
)
|
||||
{
|
||||
MainTimeProvider.StopProvidingTime();
|
||||
|
||||
// GraphicFramework.glfwTerminate();
|
||||
Environment.Exit(exitCode);
|
||||
}
|
||||
|
||||
public static void Track(
|
||||
object target,
|
||||
string message,
|
||||
bool isConsole,
|
||||
[CallerFilePath] string filePath = "",
|
||||
[CallerMemberName] string callerName = "",
|
||||
[CallerLineNumber] int lineNumber = 0
|
||||
)
|
||||
{
|
||||
Type t = target.GetType();
|
||||
string log = $"Tracked object: {nameof(target)}\t\tType:{t.Name}\n";
|
||||
FieldInfo[] fieldInfo = t.GetFields(
|
||||
BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance);
|
||||
log += "|\tFieldName\t|\tFieldType\t|\tFieldValue\t|\n";
|
||||
foreach (FieldInfo field in fieldInfo) {
|
||||
log += $"|{field.Name}\t\t|{field.FieldType}\t\t|{field.GetValue(target)}\t\t|\n";
|
||||
}
|
||||
log += $"Tracking at:\n{filePath}\t{callerName}\tline {lineNumber}\n";
|
||||
log += $"Additional information:\n{message}\n";
|
||||
if (isConsole)
|
||||
{
|
||||
Console.WriteLine(log);
|
||||
} else
|
||||
{
|
||||
DefaultLog.InternalWriteLog(log, LogType.Info);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Write a log message to log file, and mark the _level of this message.
|
||||
/// </summary>
|
||||
/// <param _label="message">
|
||||
/// A string contains information of how current evironment runs.
|
||||
/// </param>
|
||||
/// <param _label="logType">
|
||||
/// The severity of current message.
|
||||
/// </param>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void WriteLog(
|
||||
string message,
|
||||
LogType logType
|
||||
)
|
||||
{
|
||||
DefaultLog.InternalWriteLog(message, logType);
|
||||
}
|
||||
|
||||
internal static TDele ConvertAPI<TDele>(
|
||||
IntPtr funcPtr
|
||||
)
|
||||
{
|
||||
if (funcPtr == IntPtr.Zero) {
|
||||
throw new InvalidCastException(
|
||||
"Failed to convert api, the function pointer is zero.");
|
||||
}
|
||||
try
|
||||
{
|
||||
return Marshal.GetDelegateForFunctionPointer<TDele>(funcPtr);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
internal static Delegate ConvertAPI(
|
||||
IntPtr funcPtr,
|
||||
Type T
|
||||
)
|
||||
{
|
||||
if (funcPtr == IntPtr.Zero) {
|
||||
throw new InvalidCastException(
|
||||
"Failed to convert api, the function pointer is zero.");
|
||||
}
|
||||
try
|
||||
{
|
||||
return Marshal.GetDelegateForFunctionPointer(funcPtr, T);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
internal static unsafe void InternalInitialize()
|
||||
{
|
||||
// Initialize time provider and start providing time
|
||||
MainTimeProvider.StartProvidingTime();
|
||||
DefaultLog.InternalAppendLog("\n\n\n");
|
||||
DefaultLog.InternalWriteLog(
|
||||
$"Program {Assembly.GetEntryAssembly()!.FullName} on {Environment.MachineName} Started.", LogType.Info);
|
||||
MainThreadID = System.Environment.CurrentManagedThreadId;
|
||||
|
||||
/**/
|
||||
Dialogue.NeedShowWhenStartup = false;
|
||||
Dialogue.LoadDialogue();
|
||||
if (Dialogue.NeedShowWhenStartup) {
|
||||
Dialogue.CreateDialogueProcessIfNotExist();
|
||||
}
|
||||
|
||||
Numa.Initialize();
|
||||
/**/
|
||||
}
|
||||
|
||||
internal class Initializer : IStaticModule
|
||||
{
|
||||
|
||||
public string ModuleName => "StgSharpCore";
|
||||
|
||||
public void InitializeModule(
|
||||
IModuleInitializeProfile profile
|
||||
)
|
||||
{
|
||||
World.InternalInitialize();
|
||||
}
|
||||
|
||||
public void UninitializeModule()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Global"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
global using StgSharp.Internal;
|
||||
|
||||
global using System;
|
||||
global using System.Runtime.CompilerServices;
|
||||
global using System.Threading;
|
||||
|
||||
[assembly: InternalsVisibleTo("StgSharp.HighPerformance")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.RegularAnalysis")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.Graphics")]
|
||||
[assembly: InternalsVisibleTo("StgSharp.Mathematics")]
|
||||
[assembly: InternalsVisibleTo("StgSharpDebug")]
|
||||
@@ -0,0 +1,17 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="IdMultiplexer"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance
|
||||
{
|
||||
public class IdMultiplexer { }
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="BufferStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
[CollectionBuilder(typeof(BufferStackBuilder), nameof(BufferStackBuilder.Create))]
|
||||
public unsafe class BufferStack<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
private byte* _buffer;
|
||||
private ulong _count;
|
||||
|
||||
public BufferStack(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
|
||||
|
||||
AllocateBuffer(capacity);
|
||||
}
|
||||
|
||||
public int Count => (int)_count;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Clear()
|
||||
{
|
||||
((Header*)_buffer)->Top = 0;
|
||||
_count = 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Dispose()
|
||||
{
|
||||
Marshal.FreeHGlobal((nint)_buffer);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public T Pop()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top == 0) {
|
||||
throw new InvalidOperationException("Stack underflow");
|
||||
}
|
||||
|
||||
header->Top--;
|
||||
_count--;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
return dataStart[header->Top];
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Push(
|
||||
T item
|
||||
)
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top >= header->Capacity)
|
||||
{
|
||||
Expand();
|
||||
header = (Header*)_buffer;
|
||||
}
|
||||
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
dataStart[header->Top] = item;
|
||||
header->Top++;
|
||||
_count++;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryPop(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
if (header->Top == 0)
|
||||
{
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
header->Top--;
|
||||
_count--;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
value = dataStart[header->Top];
|
||||
return true;
|
||||
}
|
||||
|
||||
// Internal method for efficient bulk initialization
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal unsafe void InitializeFromSpan(
|
||||
ReadOnlySpan<T> span
|
||||
)
|
||||
{
|
||||
if (span.IsEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
Header* header = (Header*)_buffer;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
|
||||
fixed (T* sourcePtr = span) {
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart, header->Capacity * (nuint)sizeof(T),
|
||||
(nuint)span.Length * (nuint)sizeof(T));
|
||||
}
|
||||
|
||||
header->Top = (nuint)span.Length;
|
||||
_count = (ulong)span.Length;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void AllocateBuffer(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
nuint size = (nuint)sizeof(Header) + ((nuint)capacity * (nuint)sizeof(T));
|
||||
_buffer = (byte*)NativeMemory.Alloc(size);
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
header->Capacity = (nuint)capacity;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void Expand()
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
nuint newCap = header->Capacity * 2;
|
||||
nuint newSize = (nuint)sizeof(Header) + (newCap * (nuint)sizeof(T));
|
||||
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
|
||||
header = (Header*)_buffer;
|
||||
header->Capacity = newCap;
|
||||
}
|
||||
|
||||
private struct Header
|
||||
{
|
||||
|
||||
public nuint Capacity;
|
||||
public nuint Top;
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,299 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ConcurrentBufferStack"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public static class BufferStackBuilder
|
||||
{
|
||||
|
||||
// For BufferStack collection builder
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static BufferStack<T> Create<T>(
|
||||
ReadOnlySpan<T> span
|
||||
) where T : unmanaged
|
||||
{
|
||||
if (span.IsEmpty)
|
||||
{
|
||||
return new BufferStack<T>(4); // Default small capacity
|
||||
}
|
||||
|
||||
BufferStack<T> stack = new(span.Length);
|
||||
|
||||
|
||||
// Use internal method for efficient bulk initialization
|
||||
stack.InitializeFromSpan(span);
|
||||
|
||||
return stack;
|
||||
}
|
||||
|
||||
// For ConcurrentBufferStack collection builder
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static ConcurrentBufferStack<T> CreateConcurrent<T>(
|
||||
ReadOnlySpan<T> span
|
||||
) where T : unmanaged
|
||||
{
|
||||
if (span.IsEmpty)
|
||||
{
|
||||
return new ConcurrentBufferStack<T>(4); // Default small capacity
|
||||
}
|
||||
|
||||
ConcurrentBufferStack<T> stack = new(span.Length);
|
||||
|
||||
|
||||
// Use internal method for efficient bulk initialization
|
||||
stack.InitializeFromSpan(span);
|
||||
|
||||
return stack;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
[CollectionBuilder(
|
||||
builderType: typeof(BufferStackBuilder),
|
||||
methodName: nameof(BufferStackBuilder.CreateConcurrent))]
|
||||
public unsafe class ConcurrentBufferStack<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
private byte* _buffer;
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private long _count;
|
||||
|
||||
public ConcurrentBufferStack()
|
||||
{
|
||||
AllocateBuffer(4);
|
||||
}
|
||||
|
||||
public ConcurrentBufferStack(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(capacity);
|
||||
|
||||
AllocateBuffer(capacity);
|
||||
}
|
||||
|
||||
public ulong Count => (ulong)Volatile.Read(ref _count);
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
_lock.EnterBufferCopy(); // exclusive while mutating Top
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
Volatile.Write(ref _count, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_lock.Dispose();
|
||||
NativeMemory.Free(_buffer);
|
||||
}
|
||||
|
||||
public T Pop()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
_lock.EnterBufferRead(); // prevent buffer being reallocated while reading/modifying Top
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
int oldTop = Volatile.Read(ref header->Top);
|
||||
if (oldTop <= 0) {
|
||||
throw new InvalidOperationException("Stack underflow");
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
|
||||
{
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
_ = Interlocked.Decrement(ref _count);
|
||||
return dataStart[oldTop - 1];
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
|
||||
public void Push(
|
||||
T item
|
||||
)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
// Ensure capacity, expanding if needed
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
int topSnapshot = Volatile.Read(ref header->Top);
|
||||
int cap = Volatile.Read(ref header->Capacity);
|
||||
if (topSnapshot >= cap)
|
||||
{
|
||||
// Need expansion
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
header = (Header*)_buffer;
|
||||
if (header->Top >= header->Capacity)
|
||||
{
|
||||
int newCap = header->Capacity * 2;
|
||||
nuint newSize = ((nuint)sizeof(Header)) + (((nuint)newCap) * ((nuint)sizeof(T)));
|
||||
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
_buffer = (byte*)NativeMemory.Realloc(_buffer, newSize);
|
||||
((Header*)_buffer)->Capacity = newCap;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
|
||||
// Reserve and write under exclusive buffer lock to avoid consumers reading unwritten data
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
int idx = header->Top; // reserve next slot
|
||||
if (idx >= header->Capacity)
|
||||
{
|
||||
// lost race against expander or other producer; retry
|
||||
continue;
|
||||
}
|
||||
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
dataStart[idx] = item; // write first
|
||||
header->Top = idx + 1; // then publish by increasing top
|
||||
_ = Interlocked.Increment(ref _count);
|
||||
return;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public bool TryPop(
|
||||
out T value
|
||||
)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
try
|
||||
{
|
||||
Header* header = (Header*)_buffer;
|
||||
int oldTop = Volatile.Read(ref header->Top);
|
||||
if (oldTop <= 0)
|
||||
{
|
||||
value = default;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref header->Top, oldTop - 1, oldTop) == oldTop)
|
||||
{
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
value = dataStart[oldTop - 1];
|
||||
_ = Interlocked.Decrement(ref _count);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Internal method for efficient bulk initialization
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal unsafe void InitializeFromSpan(
|
||||
ReadOnlySpan<T> span
|
||||
)
|
||||
{
|
||||
if (span.IsEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
Header* header = (Header*)_buffer;
|
||||
T* dataStart = (T*)(_buffer + sizeof(Header));
|
||||
|
||||
fixed (T* sourcePtr = span) {
|
||||
Buffer.MemoryCopy(sourcePtr, dataStart,
|
||||
((nuint)header->Capacity) * ((nuint)sizeof(T)),
|
||||
((nuint)span.Length) * ((nuint)sizeof(T)));
|
||||
}
|
||||
|
||||
header->Top = span.Length;
|
||||
Volatile.Write(ref _count, span.Length);
|
||||
}
|
||||
|
||||
private void AllocateBuffer(
|
||||
int capacity
|
||||
)
|
||||
{
|
||||
nuint size = ((nuint)sizeof(Header)) + (((nuint)capacity) * ((nuint)sizeof(T)));
|
||||
_buffer = (byte*)NativeMemory.Alloc(size);
|
||||
Header* header = (Header*)_buffer;
|
||||
header->Top = 0;
|
||||
header->Capacity = capacity;
|
||||
}
|
||||
|
||||
private struct Header
|
||||
{
|
||||
|
||||
public int Capacity; // max capacity of the stack (number of elements)
|
||||
public int Top; // next available index in the stack
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
# TLSF (Two-Layer Segregated Fit) Allocator
|
||||
|
||||
This document describes the current StgSharp native arena allocator, `StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator`. The file name is historical; the implementation described below is TLSF rather than the older HLSF allocator.
|
||||
|
||||
## Overview
|
||||
|
||||
- Single contiguous native arena allocated up front
|
||||
- Headerless returned blocks; allocator metadata is stored in 32-byte nodes carved from the tail of the same arena
|
||||
- Requests rounded up to 64-byte granularity, with 64 bytes as the minimum block size
|
||||
- Free blocks grouped by coarse size level derived from the highest set bit of the block size
|
||||
- Exact-level allocation uses first-fit traversal inside the selected bucket; higher levels fall back to the head of the first non-empty bucket
|
||||
- `Free` performs bounded local coalescing; `Collect` performs full free-run coalescing across the arena
|
||||
- Single-threaded implementation; concurrent access requires external synchronization
|
||||
- No arena growth and no implicit fallback to `NativeMemory.AlignedAlloc` for oversized requests
|
||||
|
||||
## Quick Use
|
||||
|
||||
```csharp
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
|
||||
using var tlsf = new TwoLayerSegregatedFitAllocator(64 * 1024 * 1024);
|
||||
var handle = tlsf.Alloc(1024);
|
||||
|
||||
Span<byte> buffer = handle.AsSpan();
|
||||
buffer[0] = 42;
|
||||
|
||||
tlsf.Free(handle);
|
||||
```
|
||||
|
||||
## Constructors
|
||||
|
||||
- `TwoLayerSegregatedFitAllocator(nuint byteSize)`
|
||||
- `TwoLayerSegregatedFitAllocator(nuint byteSize, uint maxBlockSize)`
|
||||
- `TwoLayerSegregatedFitAllocator(nuint byteSize, int align)`
|
||||
- `TwoLayerSegregatedFitAllocator(nuint byteSize, int align, nuint maxBlockSize)`
|
||||
|
||||
Behavior summary:
|
||||
|
||||
- `byteSize` defines the total arena size
|
||||
- `maxBlockSize` defaults to 32 MB and defines the largest single request accepted by `Alloc` and `TryAlloc`
|
||||
- `align` is passed to the backing `NativeMemory.AlignedAlloc` call for arena allocation; request sizes inside the arena are still quantized to 64-byte blocks
|
||||
|
||||
## Public API
|
||||
|
||||
### `Alloc(nuint size)`
|
||||
|
||||
- Rounds `size` up to the next 64-byte block, with a minimum of 64 bytes
|
||||
- Throws `OverflowException` if the normalized size exceeds `maxBlockSize` or if the arena cannot provide a block
|
||||
- Returns a `Handle` containing block address and block size
|
||||
- Splits a sufficiently large tail remainder into a new free node and re-buckets that node
|
||||
|
||||
### `TryAlloc(nuint size, out Handle handle)`
|
||||
|
||||
- Uses the same size normalization and maximum block limit as `Alloc`
|
||||
- Returns `false` when the request exceeds the configured limit or when the arena is exhausted
|
||||
- Does not fall back to the system allocator
|
||||
|
||||
### `Free(Handle handle, int collectScanWindow = 4)`
|
||||
|
||||
- Marks the block as free
|
||||
- Scans up to `collectScanWindow` neighbors toward lower addresses and then up to `collectScanWindow` neighbors toward higher addresses
|
||||
- Merges any contiguous free neighbors found during those scans
|
||||
- Reclassifies the merged block back into the coarse bucket set
|
||||
|
||||
### `Collect()`
|
||||
|
||||
- Walks the full position chain from low address to high address
|
||||
- Merges consecutive free runs without the local scan-window limit used by `Free`
|
||||
- Restores larger reusable spans and improves bucket coverage after fragmentation-heavy phases
|
||||
|
||||
### `Dispose()`
|
||||
|
||||
- Releases the backing arena through the stored free callback
|
||||
|
||||
### `Handle`
|
||||
|
||||
- Readonly value type carrying the allocator metadata reference, block address, and block size
|
||||
- `Address` exposes the native pointer
|
||||
- `Size` exposes the normalized block size
|
||||
- `AsSpan(int range = -1)` exposes a span view over the block
|
||||
- The indexer provides byte-level access by offset
|
||||
|
||||
## Allocation Model
|
||||
|
||||
### Level Classification
|
||||
|
||||
- `GetLevel(size)` maps the normalized size to `57 - BitOperations.LeadingZeroCount(size)` and clamps the result to `_maxLevel`
|
||||
- Each bucket corresponds to one coarse size class rather than an exact size
|
||||
- The target bucket can therefore contain blocks of different sizes within the same coarse level
|
||||
|
||||
### Allocation Path
|
||||
|
||||
1. Normalize the requested size to a 64-byte multiple.
|
||||
2. Traverse the target bucket and select the first free block whose size is at least the requested size.
|
||||
3. If the target bucket has no suitable block, pop the head of the first non-empty higher-level bucket.
|
||||
4. If all buckets are empty, carve a fresh block from the spare region tracked by the arena sentinel.
|
||||
5. Split the tail remainder when both remainder size and metadata availability allow it.
|
||||
|
||||
### Metadata Layout
|
||||
|
||||
- `Metadata` is a 32-byte node
|
||||
- `NextNear` and `PreviousNear` form the address-ordered position chain
|
||||
- `NextLevel` and `PreviousLevel` form either the bucket chain or the recycled-metadata chain, depending on node state
|
||||
- The spare sentinel at the end of the arena tracks remaining virgin space, recycled metadata nodes, and position-chain anchors
|
||||
- Returned blocks remain headerless from the caller's point of view
|
||||
|
||||
### Coalescing Model
|
||||
|
||||
- `Free` performs bounded local coalescing to keep the hot path short
|
||||
- `Collect` performs full coalescing of consecutive free runs across the entire position chain
|
||||
- Recycled metadata nodes are returned to the spare metadata list and reused by later splits or fresh arena chunks
|
||||
|
||||
## Complexity and Runtime Characteristics
|
||||
|
||||
- `Alloc`: O(1) bucket index calculation plus target-level first-fit traversal; higher-level fallback is O(1)
|
||||
- `Free`: O(`collectScanWindow`) local merge cost
|
||||
- `Collect`: O(number of metadata nodes in the position chain)
|
||||
- Best steady-state behavior appears on repeated small and medium native allocations inside a pre-allocated arena
|
||||
|
||||
## BenchmarkDotNet (TLSF vs Libc)
|
||||
|
||||
Environment: .NET 8.0.26 (PerformanceDebug), x64 RyuJIT AVX2, Windows 11, AMD Ryzen 7 6800H, BenchmarkDotNet v0.15.8, InProcessEmitToolchain, LaunchCount=1, WarmupCount=6, IterationCount=15, InvocationCount=3.
|
||||
|
||||
Workload summary:
|
||||
|
||||
- Arena size: 512 MB
|
||||
- Arena alignment: 64 bytes
|
||||
- Allocation count per batch: 2048
|
||||
- Batch count: 256
|
||||
- Request set before normalization: 64, 100, 200, 400, 1024, 4096 bytes
|
||||
- Free order: shuffled per batch
|
||||
- `Free` local merge window: 4
|
||||
|
||||
Best observed run under the current harness:
|
||||
|
||||
| Method | Mean | Error | StdDev | Ratio | RatioSD | Allocated | Alloc Ratio |
|
||||
|------- |-----:|------:|-------:|------:|--------:|----------:|------------:|
|
||||
| Tlsf | 79.11 ns | 5.944 ns | 5.560 ns | 0.65 | 0.09 | - | NA |
|
||||
| Libc | 123.64 ns | 15.955 ns | 14.144 ns | 1.01 | 0.16 | - | NA |
|
||||
|
||||
Worst observed run under the same harness:
|
||||
|
||||
| Method | Mean | Error | StdDev | Ratio | RatioSD | Allocated | Alloc Ratio |
|
||||
|------- |-----:|------:|-------:|------:|--------:|----------:|------------:|
|
||||
| Tlsf | 82.95 ns | 6.337 ns | 5.617 ns | 0.81 | 0.07 | - | NA |
|
||||
| Libc | 102.63 ns | 8.590 ns | 7.173 ns | 1.00 | 0.09 | - | NA |
|
||||
|
||||
Interpretation:
|
||||
|
||||
- The current single-pass bucket selection fast path materially reduces allocator-side latency
|
||||
- The reported ratio still drifts across repeated compile-and-run cycles because the libc baseline also moves under the current InProcess harness
|
||||
- Recent repeated runs place the transient best case near 0.65 and the longer-running steady band near 0.75 to 0.80
|
||||
|
||||
## Operational Notes
|
||||
|
||||
- Maximum single-allocation size is fixed at construction time
|
||||
- Arena growth is not implemented
|
||||
- `Collect` is the global defragmentation pass; `Free` only performs bounded local coalescing
|
||||
- The allocator is intended for explicit native-memory workflows rather than as a drop-in general-purpose heap replacement
|
||||
@@ -0,0 +1,97 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.CacheLine"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
internal enum AllocationState : byte
|
||||
{
|
||||
|
||||
SpareOrCollect = 0,
|
||||
Free = 1,
|
||||
Alloc = 2,
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32, Pack = 32)]
|
||||
private unsafe struct CacheLineMetadata
|
||||
{
|
||||
|
||||
internal const uint MaxBufferSize = 0xFFFFFFFF - 64U;
|
||||
|
||||
[FieldOffset(0)] private readonly M256 FillingMask;
|
||||
|
||||
[FieldOffset(24)] internal nuint Origin;
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<CacheLineMetadata>();
|
||||
|
||||
#region tlsf linked list
|
||||
|
||||
[FieldOffset(0)] public short NextLevel;
|
||||
[FieldOffset(2)] public short NextNear;
|
||||
[FieldOffset(4)] public short PreviousLevel;
|
||||
[FieldOffset(6)] public short PreviousNear;
|
||||
|
||||
#endregion
|
||||
|
||||
#region buffer
|
||||
|
||||
[FieldOffset(8)] public uint PositionMask;
|
||||
[FieldOffset(12)] public uint SizeMask;
|
||||
|
||||
#endregion
|
||||
|
||||
#region mask
|
||||
|
||||
[FieldOffset(16)] internal AllocationState State;
|
||||
[FieldOffset(17)] internal byte ReversedProfile;
|
||||
[FieldOffset(18)] internal ushort CustomizeProfile;
|
||||
[FieldOffset(20)] internal int RefCount;
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
#pragma warning disable IDE0032
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
public unsafe struct Handle : IDisposable
|
||||
{
|
||||
|
||||
[FieldOffset(8)] private readonly CacheLineMetadata* _head;
|
||||
[FieldOffset(0)] private nuint _handle;
|
||||
|
||||
private Handle(
|
||||
nuint handle,
|
||||
CacheLineMetadata* cacheRefCount
|
||||
)
|
||||
{
|
||||
_handle = handle;
|
||||
_head = cacheRefCount;
|
||||
}
|
||||
|
||||
[Obsolete("You should not create a CacheLineHandle instance outside a L4", true)]
|
||||
public Handle() { }
|
||||
|
||||
public readonly nuint Address => _handle;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (_handle == 0) {
|
||||
return;
|
||||
}
|
||||
_handle = 0;
|
||||
_ = Interlocked.Decrement(ref _head->RefCount);
|
||||
}
|
||||
|
||||
}
|
||||
#pragma warning restore IDE0032
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Debug"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
/**/
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
public void DebugAgeAll()
|
||||
{
|
||||
// AgeAll();
|
||||
}
|
||||
|
||||
public bool DebugContains(
|
||||
nuint origin
|
||||
)
|
||||
{
|
||||
return _map.TryGet(origin, out _);
|
||||
}
|
||||
|
||||
public int DebugEvict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
return Evict(count);
|
||||
}
|
||||
|
||||
public unsafe DebugSnapshot GetDebugSnapshot()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
/*
|
||||
int maxId = _maxId;
|
||||
byte[] rrpv = new byte[maxId];
|
||||
byte* p = (byte*)_eviction;
|
||||
for (int i = 0; i < maxId; i++) {
|
||||
rrpv[i] = p[i];
|
||||
}
|
||||
return new DebugSnapshot(_activeCount, _aheadCount, maxId, rrpv);
|
||||
/**/
|
||||
}
|
||||
|
||||
public readonly struct DebugSnapshot
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// RRPV value for each line(0~3), length == MaxId
|
||||
/// </summary>
|
||||
public readonly byte[] RrpvSnapshot;
|
||||
/// <summary>
|
||||
/// Count of activated cache line handle
|
||||
/// </summary>
|
||||
public readonly int ActiveCount;
|
||||
|
||||
/// <summary>
|
||||
/// Count of predictor ahead of current map position(aheadCount)
|
||||
/// </summary>
|
||||
public readonly int AheadCount;
|
||||
|
||||
/// <summary>
|
||||
/// Max ID of the line being using or once used(maxId)
|
||||
/// </summary>
|
||||
public readonly int MaxId;
|
||||
|
||||
internal DebugSnapshot(
|
||||
int activeCount,
|
||||
int aheadCount,
|
||||
int maxId,
|
||||
byte[] rrpv
|
||||
)
|
||||
{
|
||||
ActiveCount = activeCount;
|
||||
AheadCount = aheadCount;
|
||||
MaxId = maxId;
|
||||
RrpvSnapshot = rrpv;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
/**/
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Evict"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
/// <param name="count">
|
||||
///
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// Count of lines not evicted
|
||||
/// </returns>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe int Evict(
|
||||
int count = 1
|
||||
)
|
||||
{
|
||||
count = (count < 0) ? 0 : count;
|
||||
int* pPtr = _predictCount, mPtr = _mapCount;
|
||||
|
||||
|
||||
int remain = count;
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if (pPtr[i] > mPtr[i])
|
||||
{
|
||||
continue;
|
||||
}
|
||||
remain -= TryEvictAt(i) ? 1 : 0;
|
||||
}
|
||||
return remain;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private unsafe bool TryEvictAt(
|
||||
int index
|
||||
)
|
||||
{
|
||||
CacheLineMetadata* head = &_head[index];
|
||||
if (Interlocked.CompareExchange(ref head->RefCount, int.MinValue, 0) == 0)
|
||||
{
|
||||
// EvictLine(i + index);
|
||||
_ = _map.TryRemove(head->Origin, out _);
|
||||
IL4Predict predict = CurrentPredict;
|
||||
predict.WriteBack(head->Origin, head->CustomizeProfile, new ReadOnlySpan<byte>((byte*)(ulong)(head->PositionMask + _baseAddress), (int)(ulong)head->SizeMask));
|
||||
_bufferAllocator.Free(index);
|
||||
_mapCount[index] = 0;
|
||||
_predictCount[index] = int.MaxValue;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Exceptions"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
#pragma warning disable CA1032 // Implement standard exception constructors
|
||||
|
||||
public sealed class L4PredictUnregisteredException : Exception
|
||||
{
|
||||
|
||||
public L4PredictUnregisteredException(
|
||||
int id
|
||||
)
|
||||
: base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { }
|
||||
|
||||
}
|
||||
|
||||
public sealed class L4PredictConflictException : Exception
|
||||
{
|
||||
|
||||
public L4PredictConflictException(
|
||||
nuint basePtr,
|
||||
ushort policy,
|
||||
int current
|
||||
)
|
||||
: base($"L4 predictor with id {current} has already registered a cache line with base pointer {basePtr} and policy {policy}.") { }
|
||||
|
||||
}
|
||||
#pragma warning restore CA1032 // Implement standard exception constructors
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Predict"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private IL4Predict CurrentPredict { get; set; }
|
||||
|
||||
public void ResetPredict(
|
||||
IL4Predict predict
|
||||
)
|
||||
{
|
||||
if (CurrentPredict is null)
|
||||
{
|
||||
CurrentPredict = predict;
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: write back all and register new predictor
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Predictor"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
[Flags]
|
||||
public enum PredictResult
|
||||
{
|
||||
|
||||
Success = 1,
|
||||
Exhausted = 2,
|
||||
|
||||
}
|
||||
|
||||
public interface IL4Predict
|
||||
{
|
||||
|
||||
PredictResult Predict(
|
||||
Span<CacheLineDescription> node,
|
||||
out int count
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
|
||||
/// name="cache" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Destination cache line buffer.
|
||||
/// </param>
|
||||
void Prefetch(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
Span<byte> cache
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
|
||||
/// </summary>
|
||||
/// <param name="origin">
|
||||
/// Source address associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="policy">
|
||||
/// Mapping policy associated with the cache line.
|
||||
/// </param>
|
||||
/// <param name="cache">
|
||||
/// Source cache line buffer.
|
||||
/// </param>
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
uint policy,
|
||||
ReadOnlySpan<byte> cache
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
public struct CacheLineDescription : IEquatable<CacheLineDescription>
|
||||
{
|
||||
|
||||
public override readonly bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return (obj is CacheLineDescription prediction) &&
|
||||
Address.Equals(prediction.Address) &&
|
||||
(MapPolicy == prediction.MapPolicy);
|
||||
}
|
||||
|
||||
public readonly bool Equals(
|
||||
CacheLineDescription other
|
||||
)
|
||||
{
|
||||
return
|
||||
(Address == other.Address) && (MapPolicy == other.MapPolicy);
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return (int)ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
public readonly nuint GetHashCodeLong()
|
||||
{
|
||||
return ComposePredictionKey64(Address, MapPolicy);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static nuint ComposePredictionKey64(
|
||||
nuint address,
|
||||
ushort mapPolicy
|
||||
)
|
||||
{
|
||||
nuint x = address;
|
||||
nuint y = mapPolicy;
|
||||
unchecked
|
||||
{
|
||||
nuint mixed = x ^ (y * ((nuint)0x9e3779b97f4a7c15UL));
|
||||
mixed ^= mixed >> 30;
|
||||
mixed *= (nuint)0xbf58476d1ce4e5b9UL;
|
||||
mixed ^= mixed >> 27;
|
||||
mixed *= (nuint)0x94d049bb133111ebUL;
|
||||
mixed ^= mixed >> 31;
|
||||
return mixed;
|
||||
}
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
public static bool operator ==(
|
||||
CacheLineDescription left,
|
||||
CacheLineDescription right
|
||||
)
|
||||
{
|
||||
return left.Equals(right);
|
||||
}
|
||||
|
||||
#pragma warning disable CA1051
|
||||
[FieldOffset(0)]public nuint Address;
|
||||
[FieldOffset(8)]public uint Size;
|
||||
[FieldOffset(12)]public ushort MapPolicy;
|
||||
[FieldOffset(14)]internal short Predictor;
|
||||
#pragma warning restore CA1051
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Prefetch"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.ComponentModel;
|
||||
using System.Numerics;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe partial class L4
|
||||
{
|
||||
|
||||
private partial void Prefetch()
|
||||
{
|
||||
if (CurrentPredict is null) {
|
||||
return;
|
||||
}
|
||||
IL4Predict predict = CurrentPredict;
|
||||
Span<CacheLineDescription> span = new(_prediction, PredictionCount);
|
||||
PredictResult result = predict.Predict(span, out int actualCount);
|
||||
_exhausted = (result & PredictResult.Exhausted) != 0;
|
||||
if (actualCount <= 0)
|
||||
{
|
||||
_exhausted = true;
|
||||
return;
|
||||
}
|
||||
int missIndex = 0;
|
||||
for (int i = 0; i < actualCount; i++)
|
||||
{
|
||||
ref CacheLineDescription handle = ref span[i];
|
||||
nuint address = handle.Address;
|
||||
if (!_map.TryGet(address, out _))
|
||||
{
|
||||
// miss and record
|
||||
span[missIndex] = handle;
|
||||
missIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
// now prefetch the misses
|
||||
for (int i = 0; i < missIndex; i++)
|
||||
{
|
||||
CacheLineMetadata* head;
|
||||
ref CacheLineDescription pHandle = ref span[i];
|
||||
nuint address = pHandle.Address;
|
||||
int index;
|
||||
if (_map.TryGet(address, out nuint handle))
|
||||
{
|
||||
head = (CacheLineMetadata*)handle;
|
||||
index = (int)(head - _head);
|
||||
_predictCount[index] = 0; // reset eviction counter
|
||||
|
||||
#region update predictor belonging
|
||||
|
||||
if (_predictCount[index] > short.MaxValue / 2)
|
||||
{
|
||||
int predictCount = Volatile.Read(ref _predictCount[index]);
|
||||
int mapCount = Volatile.Read(ref _mapCount[index]);
|
||||
int decrement = (int.Min(predictCount, mapCount) + 1) / 2;
|
||||
_ = Interlocked.Add(ref _predictCount[index], -decrement);
|
||||
_ = Interlocked.Add(ref _mapCount[index], -decrement);
|
||||
}
|
||||
_predictCount[index]++;
|
||||
|
||||
#endregion
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
int retryCount = 0, remain = missIndex - i;
|
||||
const int maxRetry = 1024;
|
||||
while (!_bufferAllocator.TryAlloc(pHandle.Size, out index))
|
||||
{
|
||||
remain = Evict(remain);
|
||||
retryCount++;
|
||||
if (retryCount > maxRetry) {
|
||||
throw new OverflowException($"Failed to evict cache line after {maxRetry} retries.");
|
||||
}
|
||||
if (retryCount > maxRetry / 2) {
|
||||
Thread.Sleep(1);
|
||||
}
|
||||
}
|
||||
head = _head + index;
|
||||
head->Origin = address;
|
||||
head->CustomizeProfile = pHandle.MapPolicy;
|
||||
head->RefCount = 0;
|
||||
|
||||
_predictCount[index] = 1;
|
||||
_mapCount[index] = 1;
|
||||
|
||||
// pre fetch them here
|
||||
predict.Prefetch(head->Origin, head->CustomizeProfile, new Span<byte>((byte*)(head->PositionMask + _baseAddress), (int)head->SizeMask));
|
||||
_ = _map.TryAddOrSet(address, (nuint)head, out _);
|
||||
}
|
||||
_ = Interlocked.Add(ref _aheadCount, actualCount);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4.Tlsf"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public partial class L4
|
||||
{
|
||||
|
||||
private unsafe class TLSF
|
||||
{
|
||||
|
||||
public const int MaxCacheLineCount = 8192;
|
||||
|
||||
public static readonly nuint RequestedAllocation = (nuint)MaxCacheLineCount * CacheLineMetadata.SizeOf;
|
||||
|
||||
private readonly short[] _bucket;
|
||||
private readonly CacheLineMetadata* _spareMemory;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
private int _metadataEnd;
|
||||
private readonly Ptr64 _basePtr;
|
||||
private readonly uint _maxAllocSize;
|
||||
private readonly uint _totalSize;
|
||||
|
||||
internal TLSF(
|
||||
uint byteSize,
|
||||
uint maxBlockSize,
|
||||
void* externalPointer
|
||||
)
|
||||
{
|
||||
_totalSize = byteSize;
|
||||
_basePtr = (Ptr64)(ulong)externalPointer;
|
||||
_align = 4096; // align of a memory page
|
||||
_maxAllocSize = uint.Max(64, (~64U) & (uint.Min(maxBlockSize, CacheLineMetadata.MaxBufferSize) + 63U));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new short[GetLevel(_maxAllocSize) + 2];
|
||||
_spareMemory = (CacheLineMetadata*)(ulong)(_basePtr + _totalSize);
|
||||
_metadataEnd = 0;
|
||||
|
||||
// init spare memory node
|
||||
|
||||
_spareMemory->PositionMask = 0;
|
||||
_spareMemory->SizeMask = _totalSize;
|
||||
_spareMemory->NextLevel = 0;
|
||||
_spareMemory->NextNear = 0;
|
||||
_spareMemory->PreviousLevel = 0;
|
||||
_spareMemory->PreviousNear = 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetLevel(
|
||||
uint size
|
||||
)
|
||||
{
|
||||
int origin = 25 - BitOperations.LeadingZeroCount(size);
|
||||
return int.Min(origin, _maxLevel);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Put a metadata node back to free list and waiting for next allocation.
|
||||
/// </summary>
|
||||
/// <param name="node">
|
||||
///
|
||||
/// </param>
|
||||
private void RecycleNode(
|
||||
CacheLineMetadata* node
|
||||
)
|
||||
{
|
||||
short nodeIdx = (short)(_spareMemory - node);
|
||||
int level = GetLevel(node->SizeMask);
|
||||
short nextLevelIdx = node->NextLevel;
|
||||
short prevLevelIdx = node->PreviousLevel;
|
||||
RemoveFromBucket(level, node, nodeIdx);
|
||||
|
||||
// remove from position chain
|
||||
short prevNearIdx = node->PreviousNear;
|
||||
short nextNearIdx = node->NextNear;
|
||||
CacheLineMetadata* prevNear = &_spareMemory[prevNearIdx];
|
||||
prevNear->NextNear = nextNearIdx;
|
||||
if (nextNearIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* nextNear = &_spareMemory[nextNearIdx];
|
||||
nextNear->PreviousNear = prevNearIdx;
|
||||
} else
|
||||
{
|
||||
_spareMemory->PreviousNear = prevNearIdx;
|
||||
}
|
||||
node->PreviousNear = 0;
|
||||
node->NextNear = 0;
|
||||
|
||||
node->PositionMask = 0;
|
||||
node->SizeMask = 0;
|
||||
node->State = AllocationState.SpareOrCollect;
|
||||
node->PreviousLevel = 0;
|
||||
|
||||
short nextIdx = _spareMemory->NextLevel;
|
||||
if (nextIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
||||
next->PreviousLevel = nodeIdx;
|
||||
}
|
||||
node->NextLevel = nextIdx;
|
||||
_spareMemory->NextLevel = nodeIdx;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RemoveFromBucket(
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
RemoveFromBucket(GetLevel(node->SizeMask), node, nodeIdx);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(
|
||||
int level,
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
short currentNextLevelIdx = node->NextLevel;
|
||||
short currentPrevLevelIdx = node->PreviousLevel;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
} else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
// Assuming node to remove is head f bucket
|
||||
private void RemoveFromBucketHead(
|
||||
int level,
|
||||
CacheLineMetadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
short currentNextLevelIdx = node->NextLevel;
|
||||
|
||||
// int currentPrevLevelIdx = 0;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
}
|
||||
/*
|
||||
else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
/**/
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* nextLevel = &_spareMemory[currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = 0;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
private bool TryAllocNewChunk(
|
||||
uint size,
|
||||
out CacheLineMetadata* meta
|
||||
)
|
||||
{
|
||||
short idx = _spareMemory->NextLevel;
|
||||
uint spareSize = _spareMemory->SizeMask;
|
||||
if (idx != 0)
|
||||
{
|
||||
if (spareSize < size)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
meta = &_spareMemory[idx];
|
||||
short nextSpareIdx = meta->NextLevel;
|
||||
if (nextSpareIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* nextSpare = &_spareMemory[nextSpareIdx];
|
||||
nextSpare->PreviousLevel = 0;
|
||||
}
|
||||
_spareMemory->NextLevel = nextSpareIdx;
|
||||
} else
|
||||
if (!TryGetNewMetadata(out meta)) {
|
||||
return false;
|
||||
}
|
||||
spareSize -= size;
|
||||
meta->SizeMask = size;
|
||||
meta->PositionMask = _spareMemory->PositionMask;
|
||||
meta->State = AllocationState.Free;
|
||||
meta->PreviousLevel = 0;
|
||||
meta->NextLevel = 0;
|
||||
_spareMemory->SizeMask = spareSize;
|
||||
_spareMemory->PositionMask += size;
|
||||
|
||||
// insert the new chunk to position chain
|
||||
short nearIdx = _spareMemory->PreviousNear;
|
||||
CacheLineMetadata* near = &_spareMemory[nearIdx];
|
||||
near->NextNear = idx;
|
||||
meta->PreviousNear = nearIdx;
|
||||
_spareMemory->PreviousNear = idx;
|
||||
meta->NextNear = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryGetNewMetadata(
|
||||
out CacheLineMetadata* meta
|
||||
)
|
||||
{
|
||||
ulong spareSize = _spareMemory->SizeMask;
|
||||
if (_metadataEnd >= MaxCacheLineCount)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
_metadataEnd++;
|
||||
meta = &_spareMemory[_metadataEnd];
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryTakeBucketEntry(
|
||||
uint minimumSize,
|
||||
out CacheLineMetadata* entry
|
||||
)
|
||||
{
|
||||
int level = GetLevel(minimumSize);
|
||||
|
||||
// first try to find in the target level
|
||||
{
|
||||
int idx = _bucket[level];
|
||||
while (idx != 0)
|
||||
{
|
||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
||||
if (candidate->SizeMask >= minimumSize)
|
||||
{
|
||||
RemoveFromBucket(level, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
idx = candidate->NextLevel;
|
||||
}
|
||||
}
|
||||
for (int i = level + 1; i < _bucket.Length; i++)
|
||||
{
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
{
|
||||
CacheLineMetadata* candidate = &_spareMemory[idx];
|
||||
RemoveFromBucketHead(i, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
#region alloc and free
|
||||
|
||||
public int Alloc(
|
||||
uint size
|
||||
)
|
||||
{
|
||||
uint allocSize = uint.Max(64, (~63U) & (size + 63));
|
||||
if (allocSize > _maxAllocSize) {
|
||||
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry)) {
|
||||
throw new OverflowException("No available ");
|
||||
}
|
||||
short entryIdx = (short)(_spareMemory - entry);
|
||||
uint entrySize = entry->SizeMask;
|
||||
if (entrySize < allocSize) {
|
||||
throw new OverflowException("Bucket returned an undersized block.");
|
||||
}
|
||||
uint remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > ulong.Min(4096UL, entrySize >> 7) &&
|
||||
TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||
{
|
||||
short newEntryIdx = (short)(_spareMemory - newEntry);
|
||||
newEntry->PositionMask = entry->PositionMask + allocSize;
|
||||
newEntry->SizeMask = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
short bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
}
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
short afterIdx = entry->NextNear;
|
||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->PositionMask - (ulong)_basePtr > _totalSize) {
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
return entryIdx;
|
||||
}
|
||||
|
||||
public void Collect()
|
||||
{
|
||||
short currentIdx = _spareMemory->NextNear;
|
||||
do
|
||||
{
|
||||
if (currentIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
CacheLineMetadata* current = &_spareMemory[currentIdx];
|
||||
if (current->State == AllocationState.Free)
|
||||
{
|
||||
// scan next
|
||||
uint size = 0;
|
||||
int nextIdx = current->NextNear;
|
||||
while (nextIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* next = &_spareMemory[nextIdx];
|
||||
if (next->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += next->SizeMask;
|
||||
RecycleNode(next);
|
||||
nextIdx = current->NextNear;
|
||||
}
|
||||
if (size > 0)
|
||||
{
|
||||
current->SizeMask += size;
|
||||
RemoveFromBucket(current, currentIdx);
|
||||
int newLevel = GetLevel(current->SizeMask);
|
||||
short bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = currentIdx;
|
||||
}
|
||||
current->NextLevel = bucketHeadIdx;
|
||||
current->PreviousLevel = 0;
|
||||
_bucket[newLevel] = currentIdx;
|
||||
}
|
||||
}
|
||||
currentIdx = current->NextNear;
|
||||
} while (true);
|
||||
}
|
||||
|
||||
public void Free(
|
||||
int handle,
|
||||
int collectScanWindow = 4
|
||||
)
|
||||
{
|
||||
CacheLineMetadata* head = &_spareMemory[handle];
|
||||
short headIdx = (short)(_spareMemory - head);
|
||||
uint size = head->SizeMask;
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// forward scan, previous direction
|
||||
int forwardIdx = head->PreviousNear;
|
||||
if (forwardIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
CacheLineMetadata* prev = &_spareMemory[forwardIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->SizeMask;
|
||||
head->PositionMask = prev->PositionMask;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// backward scan, next direction
|
||||
int nextIdx = head->NextNear;
|
||||
if (nextIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
CacheLineMetadata* prev = &_spareMemory[nextIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->SizeMask;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
|
||||
// recycle to level;
|
||||
head->SizeMask = size;
|
||||
head->State = AllocationState.Free;
|
||||
head->PreviousLevel = 0;
|
||||
int level = GetLevel(head->SizeMask);
|
||||
short bucketHeadIdx = _bucket[level];
|
||||
head->NextLevel = 0;
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = headIdx;
|
||||
head->NextLevel = bucketHeadIdx;
|
||||
}
|
||||
_bucket[level] = headIdx;
|
||||
return;
|
||||
}
|
||||
|
||||
public bool TryAlloc(
|
||||
uint size,
|
||||
out int h
|
||||
)
|
||||
{
|
||||
uint allocSize = uint.Max(64, (~63U) & (size + 63));
|
||||
if (allocSize > _maxAllocSize)
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out CacheLineMetadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry))
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
short entryIdx = (short)(_spareMemory - entry);
|
||||
uint entrySize = entry->SizeMask;
|
||||
if (entrySize < allocSize)
|
||||
{
|
||||
h = -1;
|
||||
return false;
|
||||
}
|
||||
uint remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out CacheLineMetadata* newEntry))
|
||||
{
|
||||
short newEntryIdx = (short)(_spareMemory - newEntry);
|
||||
newEntry->PositionMask = entry->PositionMask + allocSize;
|
||||
newEntry->SizeMask = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
short bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
CacheLineMetadata* bucketHead = &_spareMemory[bucketHeadIdx];
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
} else
|
||||
{
|
||||
newEntry->NextLevel = 0;
|
||||
}
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
short afterIdx = entry->NextNear;
|
||||
CacheLineMetadata* after = &_spareMemory[afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->PositionMask - (ulong)_basePtr > _totalSize)
|
||||
{
|
||||
h = default;
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
h = entryIdx;
|
||||
return true;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="L4"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System.Data;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe partial class L4 : IDisposable
|
||||
{
|
||||
|
||||
private const int CacheLineCount = 8192;
|
||||
private const nuint PageSize = 4096;
|
||||
private const int PredictionCount = 2048;
|
||||
private const int PredictionLeastAhead = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Array of metadata for each cache line, also used as metadata for tlsf allocator.
|
||||
/// </summary>
|
||||
private readonly CacheLineMetadata* _head;
|
||||
/// <summary>
|
||||
/// Count of cache lines mapped for each prediction, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _mapCount;
|
||||
/// <summary>
|
||||
/// Count of times each cache line is predicted, used for eviction policy.
|
||||
/// </summary>
|
||||
private readonly int* _predictCount;
|
||||
/// <summary>
|
||||
/// Original memory view for prediction span.
|
||||
/// </summary>
|
||||
private readonly CacheLineDescription* _prediction;
|
||||
private bool _disposed;
|
||||
private bool _exhausted;
|
||||
|
||||
private int _aheadCount;
|
||||
|
||||
private readonly int _bindedNuma;
|
||||
private volatile int _prefetching;
|
||||
private readonly nuint _baseAddress;
|
||||
private readonly SwissTable _map;
|
||||
|
||||
private readonly TLSF _bufferAllocator;
|
||||
|
||||
public L4(
|
||||
int bindedNuma,
|
||||
uint bufferSize = 8192 * 8192,
|
||||
int align = 16,
|
||||
uint maxCacheLineSize = 8192
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(bufferSize, CacheLineMetadata.MaxBufferSize);
|
||||
ArgumentOutOfRangeException.ThrowIfGreaterThan(maxCacheLineSize, CacheLineMetadata.MaxBufferSize);
|
||||
_bindedNuma = bindedNuma;
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
nuint headSize = ((nuint)Unsafe.SizeOf<CacheLineMetadata>()) * CacheLineCount; // 16 byte align
|
||||
nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLineDescription>()) * PredictionCount; //16 byte align
|
||||
nuint mapCountSize = SwissTable.RequestedNativeMemorySize; // 16 byte align
|
||||
nuint predictionCountSize = sizeof(int) * CacheLineCount; // 16 byte align
|
||||
nuint normalizedBufferSize = (bufferSize + (PageSize - 1)) & (~(PageSize - 1)); // align to 4096 byte, page size
|
||||
ByteCapacity = normalizedBufferSize;
|
||||
|
||||
AllocSize = headSize + mapCountSize + predictionSize + (predictionCountSize * 2) + normalizedBufferSize;
|
||||
|
||||
// malloc all memory at once, and free at once in Dispose.
|
||||
// This is to avoid fragmentation and improve performance.
|
||||
_baseAddress = (nuint)Numa.Alloc(bindedNuma, AllocSize);
|
||||
nuint ptr = _baseAddress;
|
||||
|
||||
// init all components with the buffer, and move the buffer pointer forward.
|
||||
nuint _buffer = _baseAddress;
|
||||
_bufferAllocator = new TLSF(bufferSize, maxCacheLineSize, (void*)_baseAddress);
|
||||
_buffer += normalizedBufferSize;
|
||||
_head = (CacheLineMetadata*)ptr;
|
||||
ptr += headSize;
|
||||
_prediction = (CacheLineDescription*)ptr;
|
||||
ptr += predictionSize;
|
||||
_map = SwissTable.Create(ptr, _ => { });
|
||||
ptr += mapCountSize;
|
||||
_predictCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
_mapCount = (int*)ptr;
|
||||
ptr += predictionCountSize;
|
||||
|
||||
M512* mPtr = (M512*)_head;
|
||||
for (uint i = 1; i < CacheLineCount; i++) {
|
||||
mPtr->BroadCastFrom<ulong>(0);
|
||||
}
|
||||
}
|
||||
|
||||
public nuint ByteCapacity { get; init; }
|
||||
|
||||
public nuint AllocSize { get; init; }
|
||||
|
||||
// public IL4Predict PrefetchPredict { get; set; }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ContinueOnce(
|
||||
Handle line
|
||||
)
|
||||
{
|
||||
if (line.Address > 0) {
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
}
|
||||
}
|
||||
|
||||
[System.Diagnostics.CodeAnalysis.SuppressMessage(
|
||||
"Design",
|
||||
"CA1063:Implement IDisposable correctly",
|
||||
Justification = """
|
||||
Only owns native memory via Numa.Free();
|
||||
no managed IDisposable fields.
|
||||
Dispose(bool) pattern is unnecessary.
|
||||
""")]
|
||||
public void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_map.Dispose();
|
||||
#pragma warning disable CA1816
|
||||
GC.SuppressFinalize(_bufferAllocator);
|
||||
#pragma warning restore CA1816
|
||||
|
||||
|
||||
// init order is fallow by align requirement of each component.
|
||||
Numa.Free((void*)_baseAddress, _bindedNuma, AllocSize);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public Handle Map(
|
||||
ref CacheLineDescription prediction
|
||||
)
|
||||
{
|
||||
if ((_aheadCount < PredictionLeastAhead) && !_exhausted)
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (_aheadCount < PredictionLeastAhead)
|
||||
{
|
||||
// do prefetch in work load now, sync
|
||||
Prefetch();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_prefetching = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
if (_map.TryGet(prediction.GetHashCodeLong(), out nuint entryHandle))
|
||||
{
|
||||
// cache hit
|
||||
CacheLineMetadata* entry = (CacheLineMetadata*)entryHandle;
|
||||
if (Interlocked.Increment(ref entry->RefCount) > 0)
|
||||
{
|
||||
_predictCount[entry - _head] = 0;
|
||||
_ = Interlocked.Decrement(ref _aheadCount);
|
||||
_ = Interlocked.Decrement(ref _mapCount[entry - _head]);
|
||||
nuint* hPtr = stackalloc nuint[2] { entry->PositionMask + _baseAddress, (nuint)entry };
|
||||
return *(Handle*)hPtr;
|
||||
} else
|
||||
{
|
||||
_ = Interlocked.Decrement(ref entry->RefCount);
|
||||
}
|
||||
}
|
||||
Thread.Sleep(0);
|
||||
}
|
||||
}
|
||||
|
||||
private partial void Prefetch();
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,762 @@
|
||||
# L4 Software Cache Design Specification
|
||||
|
||||
> Project: StgSharp
|
||||
> Author: Nitload
|
||||
> Version: Draft v0.3
|
||||
> Date: 2026-03-10
|
||||
|
||||
---
|
||||
|
||||
## 1. Purpose
|
||||
|
||||
### 1.1 Problem Background
|
||||
|
||||
StgSharp's matrix computation system is migrating from a "single global context table" to a
|
||||
"per-thread context" architecture to adapt to the scheduling characteristics of hybrid-architecture
|
||||
CPUs (e.g., Intel P-Core / E-Core). After the migration, each thread independently holds its own
|
||||
computation context, but matrix data may still be allocated by the HLSF allocator or global
|
||||
`malloc` on a remote NUMA node, resulting in significantly increased cross-node access latency.
|
||||
|
||||
At the same time, in matrix multiplication (FMA), the core row data of matrix A in the
|
||||
checkerboard-shaped kernel grid layout must be repacked (Packed) into a micro-kernel-friendly
|
||||
contiguous arrangement. In libraries such as OpenBLAS, this Pack operation is re-executed on
|
||||
every tile switch, whereas L4 can cache the Pack results to avoid repeated reordering.
|
||||
|
||||
Additionally, the segmented accumulation (segment) of matrix multiplication requires storing
|
||||
intermediate partial sums locally to avoid cross-NUMA writes to the target matrix on every
|
||||
segment. L4 can serve as a local accumulation buffer for partial sums, writing them back in a
|
||||
single operation after all segments complete locally.
|
||||
|
||||
### 1.2 Solution
|
||||
|
||||
The L4 cache is a purely software-implemented **data caching layer** logically positioned
|
||||
between the CPU hardware caches (L1/L2/L3) and main memory. Its core responsibilities:
|
||||
|
||||
1. **Pack result caching**: Cache the repack results of FMA core rows in matrix A to avoid repeated Pack operations
|
||||
2. **NUMA migration cache**: Migrate data from remote NUMA nodes to local memory and cache it, eliminating cross-node latency
|
||||
3. **Segment accumulation buffer**: Temporarily store partial sums from FMA segmented computation to reduce remote write operations
|
||||
4. **PLU intermediate data buffer**: Provide a local contiguous workspace for PLU decomposition
|
||||
|
||||
### 1.3 Design Goals
|
||||
|
||||
- Provide NUMA-friendly local data access for per-thread matrix computation contexts
|
||||
- Fixed capacity (64 MB), no runtime expansion, deterministic latency
|
||||
- Transparent pointer redirection: upper-layer code queries via original pointers; L4 returns local cache pointers
|
||||
- Cache eviction policy based on **RRPV mask + windowed OPT prediction**
|
||||
- Support both **StreamTransparent** and **CacheBuffer** working modes
|
||||
- Support **writable cache lines** that invoke registered callback functions for pre-processing and write-back upon eviction or explicit flush
|
||||
|
||||
---
|
||||
|
||||
## 2. Overall Architecture
|
||||
|
||||
### 2.1 Three-Component Model
|
||||
|
||||
```
|
||||
┌���─────────────────────────────────────────────────────────────────┐
|
||||
│ L4 │
|
||||
│ │
|
||||
│ ┌──────────────┐ ┌────────────────────┐ ┌──────────────────┐ │
|
||||
│ │ CacheLine │ │ L4 Core │ │ Predictor │ │
|
||||
│ │ Handle │ │ Cache │ │ Prefetch │ │
|
||||
│ │ │ │ │ │ Predictor │ │
|
||||
│ │ • local addr │ │ • Swiss Table │ │ • 2048-entry buf │ │
|
||||
│ │ • ref count │ │ • Slab Allocator │ │ • threshold 1024 │ │
|
||||
│ │ • Dispose │ │ • RRPV mask │ │ • windowed OPT │ │
|
||||
│ │ drops ref │ │ • SIMD eviction │ │ • active key set │ │
|
||||
│ │ │ │ • writeback mgmt │ │ │ │
|
||||
│ └──────────────┘ └────────────────────┘ └──────────────────┘ │
|
||||
│ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.2 Data Access Path
|
||||
|
||||
```
|
||||
Compute thread accessing data:
|
||||
|
||||
┌─────────────────────────────────────────────────────────┐
|
||||
│ Read-only path: L4.Map(originKey) │
|
||||
│ ├─ StreamTransparent + no repack needed? │
|
||||
│ │ └─ Yes → return original pointer directly │
|
||||
│ │ (zero overhead) │
|
||||
│ │ │
|
||||
│ └─ CacheBuffer mode │
|
||||
│ ├─ Swiss Table lookup (2.6ns, lock-free) │
|
||||
│ ├─ Hit → update RRPV=0, return CacheLine handle │
|
||||
│ └─ Miss → SpinWait for Predictor prefetch │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ Writable path: L4.MapWritable(originKey, preprocess, writeback) │
|
||||
│ ├─ Hit → return existing partial sum cache line │
|
||||
│ └─ Miss → allocate zero-initialized cache line, │
|
||||
│ register writeback callbacks │
|
||||
│ cache line marked dirty (dirty=1) │
|
||||
│ eviction invokes preprocess + writeback │
|
||||
├─────────────────────────────────────────────────────────┤
|
||||
│ After computation: │
|
||||
│ 3. CacheLine.Dispose() (refCount--) │
|
||||
│ 4. L4.StepComplete() │
|
||||
│ ├─ Predictor.AdvanceExecutionPoint() │
|
||||
│ └─ Predictor.NeedsPrefetch? │
|
||||
│ └─ Yes → CAS-acquire lock, execute prefetch │
|
||||
└─────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### 2.3 Key Parameters
|
||||
|
||||
| Parameter | Value | Description |
|
||||
|------------------------|--------------------------|------------------------------------------------------|
|
||||
| Total cache capacity | 64 MB | Per-NUMA-node contiguous memory |
|
||||
| Cache line size | 8192 Bytes | = 64 × `double MAT_KERNEL` |
|
||||
| Cache line count | 8192 | 64MB / 8192B |
|
||||
| Mapping table type | Swiss Table | 1024 buckets × 16 slots, SIMD-accelerated |
|
||||
| Cache line pool | Slab Allocator | Non-concurrent, fixed capacity, no auto-expansion |
|
||||
| Eviction policy | RRPV mask + windowed OPT | SoA layout + SSE2 batch scan |
|
||||
| Prefetch policy | Predictor windowed OPT | 2048-entry buffer, 1024-step lookahead |
|
||||
| Writeback policy | Callback-driven | Invokes registered callbacks on dirty eviction/flush |
|
||||
|
||||
---
|
||||
|
||||
## 3. Key Components
|
||||
|
||||
### 3.1 Swiss Table (Pointer Mapping Table)
|
||||
|
||||
**Responsibility**: Maintains the `original pointer → cache pointer` mapping.
|
||||
|
||||
- Key: original pointer `nint`, pointing to data allocated by HLSF/malloc on a remote node
|
||||
- Value: cache pointer `nint`, pointing to locally Slab-allocated cache line
|
||||
- Hash: hardware `CRC32C` instruction on the 64-bit pointer
|
||||
- Low 10 bits → bucket index (1024 buckets)
|
||||
- Bits 10-16 → H2 control byte (used for SIMD pre-filtering)
|
||||
- Scan: SSE2 `pmovmskb` instruction for parallel comparison of 16 control bytes
|
||||
- Tombstone mechanism: deleted entries set ctrl to `0xFE`, preserving probe chain integrity
|
||||
- Capacity: 1024 × 16 = 16,384 mappings, significantly larger than total cache line count (8192 lines),
|
||||
ensuring extremely low hash collision rate
|
||||
|
||||
**Measured Performance** (Ryzen 7 6800H, .NET 8, BenchmarkDotNet):
|
||||
|
||||
| Operation | Swiss Table | Dictionary | Speedup |
|
||||
|--------------------------|-------------|-------------|---------|
|
||||
| Zipfian Lookup (8192×) | 20.0 μs | 76.1 μs | 3.8× |
|
||||
| Uniform Lookup (8192×) | 18.6 μs | 50.6 μs | 2.7× |
|
||||
| Insert (4096 entries) | 18.5 μs | 46.5 μs | 2.5× |
|
||||
| SteadyState (4096 ops) | 25.3 μs | 68.3 μs | 2.7× |
|
||||
|
||||
Single lookup latency ≈ 2.6 ns, GC allocation = 0.
|
||||
|
||||
### 3.2 Slab Allocator (Cache Line Pool)
|
||||
|
||||
**Responsibility**: Manages allocation and reclamation of fixed-size (8192B) cache lines.
|
||||
|
||||
- Mode: **Non-concurrent, fixed capacity** (`FixedSlabAllocator<T>` mode),
|
||||
reusing the existing `SlabAllocator.FromBuffer<T>()` interface
|
||||
- No auto-expansion: capacity is fixed at initialization (64MB / 8192B = 8192 lines);
|
||||
allocation failure triggers eviction rather than expansion
|
||||
- Allocation complexity: O(1), pops from free stack
|
||||
- Reclamation complexity: O(1), pushes back to free stack
|
||||
- Alignment: all cache lines are 64B aligned, ensuring no cross-hardware-cache-line boundaries
|
||||
|
||||
### 3.3 Eviction Metadata (SoA Layout + RRPV Mask + Writeback Callbacks)
|
||||
|
||||
**Responsibility**: Tracks per-cache-line reference state, heat, and writeback information to determine eviction priority.
|
||||
|
||||
Replaces the four-level circular linked list from v0.1 with parallel arrays in Structure-of-Arrays layout:
|
||||
|
||||
```
|
||||
SoA Layout — Parallel Arrays
|
||||
|
||||
_refFlags[i] : byte Reference count snapshot (0=free, non-zero=held)
|
||||
_evictionFlags[i] : byte dirty(1bit) + RRPV(2bit) + costClass(1bit)
|
||||
_originKeys[i] : nuint Original pointer (reverse-deregisters Swiss Table entry on eviction)
|
||||
_preprocess[i] : nint Writeback pre-processing function pointer (called before dirty eviction)
|
||||
_writeback[i] : nint Writeback function pointer (called during dirty eviction)
|
||||
_callbackCtx[i] : nuint Callback context (passed to preprocess/writeback)
|
||||
```
|
||||
|
||||
**Eviction Flag Bit Layout** (`_evictionFlags[i]`):
|
||||
|
||||
The RRPV array uses byte storage for each cache line's usage state; the low 2 bits encode the RRPV value.
|
||||
|
||||
**RRPV Semantics**:
|
||||
|
||||
| RRPV | Meaning | Eviction Priority |
|
||||
|------|---------------------------------|---------------------------|
|
||||
| 0 | Will be re-accessed very soon | Lowest (do not evict) |
|
||||
| 1 | Will be accessed shortly | Low |
|
||||
| 2 | Will be accessed in a while | Medium |
|
||||
| 3 | May never be accessed again | Highest (evict first) |
|
||||
|
||||
**RRPV Update Events**:
|
||||
|
||||
| Event | Operation | Description |
|
||||
|--------------------|---------------------------------|------------------------------------------------------|
|
||||
| Load (insert) | RRPV = 0 | Freshly loaded data will be used soon |
|
||||
| Hit | `_evictionFlags[i] = 0` | Bits 1-2 cleared, RRPV reset to 0 |
|
||||
| Eviction scan | Find lines with RRPV=3 to evict | If none found, increment all RRPV +1 (aging) |
|
||||
|
||||
**Advantages over the v0.1 four-level circular linked list**:
|
||||
|
||||
| Comparison | v0.1 Circular List | v0.3 RRPV Mask + Callbacks |
|
||||
|----------------------|-------------------------------------|--------------------------------------|
|
||||
| Data structure | 4 doubly-linked lists + next ptrs | Byte array + function pointer arrays |
|
||||
| Warm-up operation | Unlink + relink (4 pointer writes) | `+= 1` (1-byte write) |
|
||||
| Aging operation | Per-node list migration | SIMD batch +1 (16 entries/cycle) |
|
||||
| Eviction scan | Iterate list head (pointer chasing) | SIMD pattern matching (16/cycle) |
|
||||
| Cache friendliness | Poor (pointer chasing) | Good (contiguous memory scan) |
|
||||
| Reference protection | None | Lines with refCount > 0 not evicted |
|
||||
| Writeback support | None | Callback-function driven |
|
||||
|
||||
### 3.4 Writeback Callback Mechanism
|
||||
|
||||
**Responsibility**: Provides flexible writeback strategies for writable cache lines, using native function pointers to avoid managed overhead.
|
||||
|
||||
**Design Principle**: All writable cache lines are treated as dirty by default. On eviction or explicit flush,
|
||||
L4 itself does not manage the writeback strategy; instead, the bound predictor is responsible for implementing it.
|
||||
|
||||
The predictor is defined via the following interface:
|
||||
|
||||
```
|
||||
public interface IPrefetchPredict
|
||||
{
|
||||
|
||||
int Predict(
|
||||
Span<CacheLinePrediction> node
|
||||
);
|
||||
|
||||
bool Prefetch(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
void WriteBack(
|
||||
nuint origin,
|
||||
ulong policy,
|
||||
nuint cache
|
||||
);
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
**Typical Callback Implementations**:
|
||||
|
||||
```
|
||||
Scenario 1: Segment accumulation (FMA partial sum)
|
||||
preprocess = null (no pre-processing needed; data is already a double array)
|
||||
writeback = AccumulateAdd (element-wise dst[i] += src[i], AVX2-vectorized)
|
||||
|
||||
Scenario 2: Writeback of packed intermediate results
|
||||
preprocess = UnpackKernelRow (reverse-repack from Pack format to row-major order)
|
||||
writeback = AccumulateAdd
|
||||
|
||||
Scenario 3: PLU decomposition intermediate data
|
||||
preprocess = null
|
||||
writeback = DirectCopy (direct overwrite memcpy)
|
||||
```
|
||||
|
||||
### 3.5 CacheLine (Cache Line Handle)
|
||||
|
||||
**Responsibility**: A cache line access handle held by the compute thread; prevents in-use lines from being evicted via reference counting.
|
||||
|
||||
```csharp
|
||||
public ref struct CacheLine : IDisposable
|
||||
{
|
||||
private ref int _activeCount; // L4 global active count
|
||||
private ref int _refCount; // Per-line reference count
|
||||
private nuint _handle; // Local cache data address
|
||||
|
||||
// Construction: Interlocked.Increment(ref _activeCount)
|
||||
// Interlocked.Increment(ref _refCount)
|
||||
// Dispose: Interlocked.Decrement on both counters
|
||||
}
|
||||
```
|
||||
|
||||
- `ref struct` ensures it cannot escape to the heap
|
||||
- Reference counts use `Interlocked` atomic operations for thread safety
|
||||
- Lines with `_refCount > 0` are skipped during eviction scans
|
||||
- StreamTransparent mode returns a CacheLine that does not occupy a cache slot
|
||||
- Both read-only and writable paths return the same CacheLine type,
|
||||
callers decide read/write behavior via `.Address`
|
||||
|
||||
### 3.6 Predictor (Prefetch Predictor)
|
||||
|
||||
**Responsibility**: Provides deterministic prefetch predictions for L4 based on the windowed OPT strategy.
|
||||
|
||||
**Core Principle**: Matrix computation task sequences are deterministic — the scheduler pre-allocates
|
||||
task blocks `(kr, kc, zz)` for each thread, allowing precise computation of the cache line keys
|
||||
required at each future step. This gives L4 a predictive capability that hardware caches cannot
|
||||
possess: **direct visibility into the future**, rather than inferring the future from history as
|
||||
strategies like Hawkeye do.
|
||||
|
||||
```
|
||||
Predictor Sliding Window:
|
||||
|
||||
Task sequence: ... [T_now] [T+1] [T+2] ... [T+1023] ... [T+2047]
|
||||
↑ execution point ↑ buffer tail
|
||||
|
||||
Window capacity = 2048 prediction entries
|
||||
Prefetch trigger threshold = trigger refill when prefetch frontier
|
||||
leads execution point by fewer than 1024 steps
|
||||
Refill batch size = 64 entries (avoids excessive single-refill latency)
|
||||
```
|
||||
|
||||
**数据结构**:
|
||||
|
||||
```csharp
|
||||
internal class Predictor
|
||||
{
|
||||
private const int WindowCapacity = 2048;
|
||||
private const int RefillThreshold = 1024;
|
||||
|
||||
private PrefetchEntry[] _buffer; // Circular prediction buffer
|
||||
private int _head; // Prefetch frontier (prefetched up to here)
|
||||
private int _tail; // Production position (scheduler fills up to here)
|
||||
private int _executed; // Execution point (compute thread has completed up to here)
|
||||
|
||||
private HashSet<nuint> _activeKeys; // All active keys within the window (deduplicated)
|
||||
}
|
||||
|
||||
struct PrefetchEntry
|
||||
{
|
||||
public nuint Key; // Cache line identifier
|
||||
public nuint SourceAddress; // Source data address
|
||||
public bool NeedsRepack; // Whether non-contiguous repacking is needed
|
||||
}
|
||||
```
|
||||
|
||||
**Cooperation with Eviction Policy**:
|
||||
|
||||
```
|
||||
Eviction priority (from highest to lowest priority):
|
||||
|
||||
1. refCount==0, not in window, RRPV==3, dirty==0 ← First (zero-cost eviction)
|
||||
2. refCount==0, not in window, RRPV==3, dirty==1 ← Second (needs writeback but cold)
|
||||
3. refCount==0, not in window, RRPV<3, dirty==0 ← Ordered by RRPV
|
||||
4. refCount==0, not in window, RRPV<3, dirty==1 ← Ordered by RRPV
|
||||
5. refCount==0, in window ← Avoid evicting if possible
|
||||
6. refCount>0 ← Never evict
|
||||
```
|
||||
|
||||
Windowed OPT means most eviction decisions do not depend on RRPV — "not in window" alone is
|
||||
sufficient to determine evictability. RRPV provides fine-grained differentiation among
|
||||
out-of-window lines; the dirty bit preferentially evicts clean lines at equal RRPV to avoid writeback overhead.
|
||||
|
||||
### 3.7 Concurrency Model
|
||||
|
||||
L4 uses a **no-extra-threads** design; all operations are performed by compute threads themselves:
|
||||
|
||||
| Operation | Executor | Concurrency |
|
||||
|------------------------------------|---------------------------------|--------------------------------------|
|
||||
| Lookup (Swiss Table read) | Any compute thread | ✅ Fully parallel, lock-free |
|
||||
| HIT RRPV update | Any compute thread | ✅ Single-byte write, no sync needed |
|
||||
| refCount increment/decrement | Any compute thread | ✅ Interlocked atomic operations |
|
||||
| Prefetch refresh (eviction + load) | CAS-winner thread | Serial (only one thread enters) |
|
||||
| Dirty writeback | CAS-winner thread | Serial (within eviction flow) |
|
||||
| FlushAllDirty | Single thread after computation | Serial (no concurrency) |
|
||||
|
||||
**Prefetch Refresh CAS Lock**:
|
||||
|
||||
```csharp
|
||||
private int _refreshLock; // 0=free, 1=locked
|
||||
|
||||
private void TryRefreshPrefetch()
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _refreshLock, 1, 0) != 0)
|
||||
return; // Another thread is doing it; current thread continues computing
|
||||
try { /* eviction + prefetch */ }
|
||||
finally { Volatile.Write(ref _refreshLock, 0); }
|
||||
}
|
||||
```
|
||||
|
||||
**Swiss Table Concurrent Read Safety Guarantee**: The prefetch refresh thread follows strict write ordering
|
||||
on insertion: write Slab data first, then write eviction metadata and callback pointers, `sfence`, then
|
||||
write the Swiss Table control byte and key/value, ensuring that when a compute thread reads the control
|
||||
byte, the data is already fully visible.
|
||||
|
||||
---
|
||||
|
||||
## 4. Operation Flows
|
||||
|
||||
### 4.1 Cache Lookup (Read-Only Hit)
|
||||
|
||||
1. Compute CRC32C hash of the original pointer
|
||||
2. Swiss Table lookup: SIMD scan ctrl → compare key → return cache pointer
|
||||
3. Update RRPV: `_evictionFlags[slot] &= 0xF9` (RRPV zeroed, dirty bit unchanged)
|
||||
4. Construct CacheLine handle (refCount++)
|
||||
5. Return CacheLine; upper-layer code accesses local data via `.Address`
|
||||
|
||||
### 4.2 Cache Lookup (Writable Hit)
|
||||
|
||||
1. Swiss Table lookup hits
|
||||
2. Update RRPV: `_evictionFlags[slot] &= 0xF9`
|
||||
3. Construct CacheLine handle (refCount++)
|
||||
4. Return CacheLine; upper-layer code reads/writes the cache line via `.Address` (e.g., partial sum +=)
|
||||
|
||||
### 4.3 Cache Miss (Read-Only)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Compute thread enters `Thread.Sleep(0)` spin loop
|
||||
3. Wait for Predictor-driven prefetch refresh to load the line into L4
|
||||
4. Exit wait loop when Swiss Table shows a hit
|
||||
5. Construct CacheLine handle and return
|
||||
|
||||
Under normal operation, the Predictor always leads the execution point by 1024+ steps,
|
||||
misses occur only during cold start or extreme bursts.
|
||||
|
||||
### 4.4 Cache Miss (Writable)
|
||||
|
||||
1. Swiss Table lookup misses
|
||||
2. Allocate a cache line from Slab
|
||||
3. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
4. Zero-initialize the cache line (`Unsafe.InitBlock(dest, 0, 8192)`)
|
||||
5. Register writeback callbacks: `_preprocess[slot]`, `_writeback[slot]`, `_callbackCtx[slot]`
|
||||
6. Set eviction flags: `dirty=1, RRPV=0`
|
||||
7. Write `_originKeys[slot]`
|
||||
8. `sfence`
|
||||
9. Write Swiss Table mapping
|
||||
10. Construct CacheLine handle and return
|
||||
|
||||
### 4.5 Prefetch Refresh
|
||||
|
||||
Triggered by compute threads in `StepComplete()`; CAS lock ensures single-thread execution:
|
||||
|
||||
1. Synchronize refFlags snapshot (`_refCounts[i] > 0 → _refFlags[i] = 1`)
|
||||
2. Retrieve pending prefetch entries from Predictor (up to 64 at a time)
|
||||
3. For each entry:
|
||||
- Swiss Table hit → mark RRPV=0 (hot-line protection)
|
||||
- Swiss Table miss → execute load:
|
||||
a. Allocate cache line from Slab
|
||||
b. If Slab has no free slots → trigger eviction (see 4.6)
|
||||
c. Execute Pack reorder or memcpy based on `NeedsRepack` flag
|
||||
d. Write eviction metadata (originKey, RRPV=0, dirty=0, callbacks=null)
|
||||
e. `sfence`
|
||||
f. Write Swiss Table mapping
|
||||
|
||||
### 4.6 Cache Eviction (SIMD Scan + Writeback)
|
||||
|
||||
```
|
||||
Eviction scan pseudocode:
|
||||
|
||||
Pass 1: Find refCount==0, RRPV==3, dirty==0 (zero-cost eviction)
|
||||
for i in range(scanCursor, CacheLineCount), step=16:
|
||||
refChunk = SSE2.Load(refFlags + i)
|
||||
refFreeMask = SSE2.MoveMask(SSE2.CompareEqual(refChunk, zero))
|
||||
|
||||
evChunk = SSE2.Load(evictionFlags + i)
|
||||
// RRPV==3 and dirty==0 → byte & 0x07 == 0x06
|
||||
masked = SSE2.And(evChunk, 0x07)
|
||||
coldCleanMask = SSE2.MoveMask(SSE2.CompareEqual(masked, 0x06))
|
||||
|
||||
candidates = refFreeMask & coldCleanMask
|
||||
while candidates != 0:
|
||||
bit = TrailingZeroCount(candidates)
|
||||
idx = i + bit
|
||||
if originKeys[idx] not in Predictor.ActiveKeys:
|
||||
evict(idx) // No writeback, discard directly
|
||||
return
|
||||
candidates &= candidates - 1
|
||||
|
||||
Pass 2: Find refCount==0, RRPV==3, dirty==1 (needs writeback)
|
||||
Same as above, but match byte & 0x07 == 0x07
|
||||
evict triggers writeback callbacks
|
||||
|
||||
Pass 3: Increment all RRPV+1 (aging), retry
|
||||
Up to 3 rounds
|
||||
```
|
||||
|
||||
**逐出执行**:
|
||||
|
||||
```csharp
|
||||
private void Evict(int slot)
|
||||
{
|
||||
byte flags = _evictionFlags[slot];
|
||||
|
||||
// Dirty writeback
|
||||
if ((flags & 0x01) != 0)
|
||||
{
|
||||
nuint origin = _originKeys[slot];
|
||||
nuint cached = SlotAddressFromIndex(slot);
|
||||
nuint ctx = _callbackCtx[slot];
|
||||
|
||||
// 1. Pre-processing (optional)
|
||||
var preprocess = (delegate* unmanaged<nuint, nuint, void>)_preprocess[slot];
|
||||
if (preprocess != null)
|
||||
preprocess(cached, ctx);
|
||||
|
||||
// 2. Writeback
|
||||
var writeback = (delegate* unmanaged<nuint, nuint, nuint, void>)_writeback[slot];
|
||||
if (writeback != null)
|
||||
writeback(origin, cached, ctx);
|
||||
}
|
||||
|
||||
// Standard eviction
|
||||
_map.TryRemove(_originKeys[slot], out _);
|
||||
_slab.Free(SlotAddressFromIndex(slot));
|
||||
_evictionFlags[slot] = 0;
|
||||
_originKeys[slot] = 0;
|
||||
_preprocess[slot] = 0;
|
||||
_writeback[slot] = 0;
|
||||
_callbackCtx[slot] = 0;
|
||||
}
|
||||
```
|
||||
|
||||
### 4.7 显式刷新
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// Explicitly writes back the specified dirty cache line. Called after all segments complete.
|
||||
/// </summary>
|
||||
public void FlushDirty(nuint originKey)
|
||||
{
|
||||
if (_map.TryGet(originKey, out nuint addr))
|
||||
{
|
||||
int slot = SlotIndex(addr);
|
||||
if ((_evictionFlags[slot] & 0x01) != 0)
|
||||
{
|
||||
// Execute pre-processing + writeback
|
||||
InvokeWriteBack(slot);
|
||||
_evictionFlags[slot] &= 0xFE; // Clear dirty bit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes back all dirty cache lines in bulk. Called after entire matrix computation completes.
|
||||
/// </summary>
|
||||
public void FlushAllDirty()
|
||||
{
|
||||
for (int i = 0; i < CacheLineCount; i++)
|
||||
{
|
||||
if ((_evictionFlags[i] & 0x01) != 0 && _originKeys[i] != 0)
|
||||
{
|
||||
InvokeWriteBack(i);
|
||||
_evictionFlags[i] &= 0xFE;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4.8 Working Mode Switch
|
||||
|
||||
```csharp
|
||||
public enum L4Mode
|
||||
{
|
||||
/// <summary>
|
||||
/// Pass-through mode: Map() returns the original pointer directly; L4 does not intervene.
|
||||
/// Suitable for single-NUMA + no-repack scenarios (e.g., vector addition, scalar multiplication).
|
||||
/// </summary>
|
||||
StreamTransparent,
|
||||
|
||||
/// <summary>
|
||||
/// Buffer mode: full lookup/Pack/cache/eviction pipeline.
|
||||
/// Suitable for FMA scenarios requiring repacking or multi-NUMA scenarios.
|
||||
/// </summary>
|
||||
CacheBuffer
|
||||
}
|
||||
```
|
||||
|
||||
Mode is determined at construction time and does not change at runtime. Decision logic:
|
||||
|
||||
- Requires Pack reordering (FMA core rows of matrix A) → CacheBuffer
|
||||
- Multiple NUMA nodes → CacheBuffer
|
||||
- Requires segment accumulation → CacheBuffer
|
||||
- Single NUMA + no reordering + no segment → StreamTransparent
|
||||
|
||||
---
|
||||
|
||||
## 5. Segment Accumulation Scenario
|
||||
|
||||
### 5.1 Problem
|
||||
|
||||
For matrix multiplication `C = A × B` when the K dimension is large, segmented accumulation is required:
|
||||
|
||||
```
|
||||
segment 0: C_partial = A[kr, 0..KC] × B[0..KC, kc]
|
||||
segment 1: C_partial += A[kr, KC..2KC] × B[KC..2KC, kc]
|
||||
...
|
||||
segment N: C_partial += A[kr, N*KC..K] × B[N*KC..K, kc]
|
||||
|
||||
Final: C[kr, kc] = Σ all segment partial sums
|
||||
```
|
||||
|
||||
If each segment's result is `+=` directly to the C matrix,
|
||||
and C resides on a remote NUMA node, every segment incurs cross-node write latency.
|
||||
|
||||
### 5.2 L4 Solution
|
||||
|
||||
Temporarily store C's partial sums in L4's writable cache lines:
|
||||
|
||||
```
|
||||
segment 0: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// First miss → allocate zero-initialized cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
segment 1: c = l4.MapWritable(C_key, null, &AccumulateAdd)
|
||||
// Hit! Returns existing partial sum cache line
|
||||
FMA(c.Address, a.Address, b.Address) // c += a*b
|
||||
c.Dispose()
|
||||
|
||||
... after all segments complete ...
|
||||
|
||||
l4.FlushDirty(C_key)
|
||||
// Calls AccumulateAdd(C_origin, cacheline, ctx)
|
||||
// → Adds local partial sum back to remote C matrix in one shot
|
||||
```
|
||||
|
||||
### 5.3 Benefits
|
||||
|
||||
```
|
||||
Without segment buffering (OpenBLAS approach):
|
||||
Each segment writes C matrix across NUMA
|
||||
Remote writes per output tile = Zk times
|
||||
Per-write latency = ~200-300ns (cross-node)
|
||||
|
||||
With L4 segment buffering:
|
||||
All segment += operations complete locally in L4 cache lines
|
||||
Partial sums likely remain in L1/L2 (8KB << L1 capacity)
|
||||
Remote writes per output tile = 1 (final FlushDirty)
|
||||
Writeback uses AVX2-vectorized stores: 8192B = 32 × 256-bit stores
|
||||
|
||||
Reduction ratio = Zk : 1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Uniqueness and Advantages
|
||||
|
||||
### 6.1 Deterministic Prefetch (Windowed OPT)
|
||||
|
||||
Essential difference from hardware cache replacement policies (LRU, RRIP, Hawkeye):
|
||||
|
||||
| Policy | Information Source | Prediction Method | Accuracy |
|
||||
|---------------------|-------------------------|-------------------------------|---------------------------|
|
||||
| LRU | Past access order | Least recently used | Moderate |
|
||||
| RRIP | Insert/hit markers | Predict reuse distance | Moderate-High |
|
||||
| Hawkeye | OPTgen retrospection | Learn past to infer future | High |
|
||||
| **L4 Predictor** | **Future task queue** | **Direct future visibility** | **Perfect** (within window) |
|
||||
|
||||
L4's Predictor is a **deterministic Oracle** — it does not predict; it knows.
|
||||
Because matrix computation task sequences are fully determined at scheduler allocation time.
|
||||
|
||||
### 6.2 NUMA Adaptability
|
||||
|
||||
The same code automatically gains NUMA benefits on different hardware configurations:
|
||||
|
||||
| Hardware Configuration | L4 Behavior |
|
||||
|-------------------------|-------------------------------------------------------------|
|
||||
| Single-socket CPU | Pack cache + segment accumulation buffer |
|
||||
| Dual-socket CPU | Pack cache + NUMA migration cache + segment accum. |
|
||||
| Hybrid-architecture CPU | Pack cache + cross-domain migration cache + segment accum. |
|
||||
|
||||
### 6.3 Callback-Driven Writeback Strategy
|
||||
|
||||
Using native function pointers for writeback avoids:
|
||||
|
||||
- Virtual method call overhead
|
||||
- Interface dispatch overhead
|
||||
- Delegate GC pressure
|
||||
- Hard-coding specific writeback logic
|
||||
|
||||
Callers register `preprocess` and `writeback` callbacks at `MapWritable` time,
|
||||
L4 invokes them directly through function pointers on eviction or Flush,
|
||||
**zero managed overhead, zero virtual dispatch, performance equivalent to C function calls**.
|
||||
|
||||
### 6.4 Reference Count Protection
|
||||
|
||||
Through `CacheLine` handle reference counting, cache lines actively in use by compute threads are
|
||||
absolutely never evicted (including writeback not being triggered). The SIMD eviction scan skips
|
||||
lines with `refCount > 0` immediately via the `_refFlags` array, without checking linked lists or acquiring locks.
|
||||
|
||||
### 6.5 Cache Line Size Aligned to Matrix Kernel
|
||||
|
||||
The 8192B cache line size is not arbitrary:
|
||||
|
||||
- 1 `double MAT_KERNEL` (4×4 double matrix) = 128 Bytes
|
||||
- 1 cache line = 64 `MAT_KERNEL` units
|
||||
- Matrix multiplication micro-kernels operate on data sizes perfectly aligned to cache lines
|
||||
- Compute kernel functions operate on cache lines as units; no cross-cache-line accesses occur
|
||||
|
||||
### 6.6 Fixed-Capacity Determinism
|
||||
|
||||
- Slab allocator never expands → memory footprint upper bound is known (64 MB)
|
||||
- SIMD eviction scan → latency upper bound is known (at most 3 aging rounds × 512 SIMD comparisons)
|
||||
- Swiss Table fixed 16,384 slots → lookup latency upper bound is known
|
||||
- Writeback callback latency is controlled by the caller → predictable
|
||||
- Worst-case L4 latency = lookup + eviction (incl. writeback) + allocation + memcpy, all statically computable
|
||||
|
||||
### 6.7 Reuse of Existing StgSharp Infrastructure
|
||||
|
||||
| L4 Component | Reused StgSharp Infrastructure |
|
||||
|-----------------------|-------------------------------------------------------|
|
||||
| Mapping table | `SwissTable` (implemented and benchmark-validated) |
|
||||
| Mapping table SIMD | `IBucketScanner` + `SIMDID` architecture detection |
|
||||
| Cache line alloc | `SlabAllocator.FromBuffer<T>()` |
|
||||
| Hash computation | Hardware `CRC32C` instruction |
|
||||
| Memory allocation | `NativeMemory.AlignedAlloc` |
|
||||
| Future multi-platform | `IBucketScanner` + NRGA compilation pipeline |
|
||||
|
||||
### 6.8 Originality Analysis
|
||||
|
||||
After surveying existing mainstream numerical computation libraries, L4's design has
|
||||
**no direct precedent or equivalent in known public implementations**.
|
||||
|
||||
#### NUMA Strategy Comparison with Existing Libraries
|
||||
|
||||
| Library / Framework | NUMA Strategy | Software Cache Layer | Mapping Table | Eviction Policy | Dirty Writeback |
|
||||
|---------------------|-------------------------------------|---------------------|-----------------|-------------------------|--------------------|
|
||||
| Intel MKL | Thread pinning + `numactl` | None | None | None | None |
|
||||
| OpenBLAS | Thread affinity + block alignment | None | None | None | None |
|
||||
| BLIS | Multi-level block alignment L1/L2/L3| None | None | None | None |
|
||||
| Eigen | No explicit NUMA handling | None | None | None | None |
|
||||
| cuBLAS / rocBLAS | GPU memory (hardware-solved) | None | None | None | None |
|
||||
| **StgSharp L4** | **Software-managed thread-local cache** | **Yes** | **Swiss Table** | **RRPV + windowed OPT** | **Callback-driven** |
|
||||
|
||||
---
|
||||
|
||||
## 7. 从 v0.1 到 v0.3 的变更摘要
|
||||
|
||||
| 项目 | v0.1 | v0.3 |
|
||||
|-----------------|-------------------------------|---------------------------------------|
|
||||
| 逐出数据结构 | 4 级环形双向链表 | RRPV 2-bit 掩码 + 8KB SoA 数组 |
|
||||
| 逐出扫描方式 | 链表头指针遍历 | SSE2 批量模式匹配(16 行/周期) |
|
||||
| 热���更新 | 链表节点摘插(4 指针写) | 单字节位运算 `&= 0x03` |
|
||||
| 老化操作 | 逐节点移链表 | SIMD 批量 +1 |
|
||||
| 引用保护 | 无 | refCount > 0 时 SIMD 扫描跳过 |
|
||||
| 预取策略 | 无 | Predictor 窗口 OPT(2048 条缓冲) |
|
||||
| 线程模型 | L4 服务线程 + Command Queue | 无额外线程,CAS 抢锁 |
|
||||
| 命令队列 | 链式 CommandCache 分段 | 不需要(预取刷新替代命令驱动) |
|
||||
| 缓存行元数据 | Slab 分配的 CacheLineHead 结构 | SoA 平行数组 + 函数指针数组 |
|
||||
| 工作模式 | 单一模式 | StreamTransparent / CacheBuffer 切换 |
|
||||
| 写回策略 | 无(只读缓存) | dirty 位 + preprocess/writeback 回调 |
|
||||
| 写回机制 | 无 | 本机函数指针,零托管开销 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 未来工作
|
||||
|
||||
- [ ] L4 根类型重构(组合 Swiss Table、Slab、SoA 元数据、Predictor)
|
||||
- [ ] Predictor 与矩阵乘法调度器的集成
|
||||
- [ ] Segment 累加写回的 AVX2 向量化实现
|
||||
- [ ] SIMD 逐出扫描的 AVX2 升级(32 行/周期)
|
||||
- [ ] Swiss Table ARM NEON 实现
|
||||
- [ ] Swiss Table AVX-512 升级(64 槽扫描从 16 次降至 4 次)
|
||||
- [ ] 墓碑累积监控与自动 rehash 策略
|
||||
- [ ] 性能基准测试(NUMA 本地 vs 远端访问延迟对比)
|
||||
- [ ] Segment 累加 vs 直接写回的 benchmark 对比
|
||||
- [ ] 与逐线程矩阵上下文的集成验证
|
||||
- [ ] 双路 6242R 环境下的 NUMA 搬运性能验证
|
||||
- [ ] 远期:PPC64LE 平台的 VSX 原生 Swiss Table 扫描支持
|
||||
|
||||
---
|
||||
|
||||
## 9. 参考
|
||||
|
||||
- Google Abseil Swiss Tables Design Notes
|
||||
- HP Labs RRIP (Re-Reference Interval Prediction) 论文
|
||||
- Hawkeye: Efficient Cache Replacement via OPTgen (ISCA 2017)
|
||||
- StgSharp HLSF Allocator 设计文档
|
||||
- StgSharp SIMD Architecture (`SIMDID` 体系)
|
||||
- StgSharp NRGA Roadmap
|
||||
- StgSharp `SlabAllocator<T>` 实现
|
||||
- StgSharp `SwissTable` 实现及 Benchmark
|
||||
@@ -0,0 +1,364 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Ptr64"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
#pragma warning disable CA2225
|
||||
|
||||
public readonly struct Ptr64 : IEquatable<Ptr64>, IComparable<Ptr64>, IFormattable
|
||||
{
|
||||
|
||||
public Ptr64(
|
||||
ulong value
|
||||
)
|
||||
{
|
||||
Value = value;
|
||||
}
|
||||
|
||||
public ulong Value { get; }
|
||||
|
||||
public int CompareTo(
|
||||
Ptr64 other
|
||||
)
|
||||
{
|
||||
return Value.CompareTo(other.Value);
|
||||
}
|
||||
|
||||
public bool Equals(
|
||||
Ptr64 other
|
||||
)
|
||||
{
|
||||
return Value == other.Value;
|
||||
}
|
||||
|
||||
public override bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return obj is Ptr64 other && Equals(other);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return Value.GetHashCode();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return Value.ToString();
|
||||
}
|
||||
|
||||
public string ToString(
|
||||
string? format,
|
||||
IFormatProvider? formatProvider
|
||||
)
|
||||
{
|
||||
return Value.ToString(format, formatProvider);
|
||||
}
|
||||
|
||||
public static Ptr64 operator -(
|
||||
Ptr64 left,
|
||||
ulong right
|
||||
)
|
||||
{
|
||||
return new(left.Value - right);
|
||||
}
|
||||
|
||||
public static ulong operator -(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value - right.Value;
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value != right.Value;
|
||||
}
|
||||
|
||||
public static Ptr64 operator &(
|
||||
Ptr64 left,
|
||||
ulong mask
|
||||
)
|
||||
{
|
||||
return new(left.Value & mask);
|
||||
}
|
||||
|
||||
public static Ptr64 operator |(
|
||||
Ptr64 left,
|
||||
ulong mask
|
||||
)
|
||||
{
|
||||
return new(left.Value | mask);
|
||||
}
|
||||
|
||||
public static Ptr64 operator +(
|
||||
Ptr64 left,
|
||||
ulong right
|
||||
)
|
||||
{
|
||||
return new(left.Value + right);
|
||||
}
|
||||
|
||||
public static bool operator <(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value < right.Value;
|
||||
}
|
||||
|
||||
public static Ptr64 operator <<(
|
||||
Ptr64 left,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
return new Ptr64(left.Value << shift);
|
||||
}
|
||||
|
||||
public static bool operator <=(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value <= right.Value;
|
||||
}
|
||||
|
||||
public static bool operator ==(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value == right.Value;
|
||||
}
|
||||
|
||||
public static bool operator >(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value > right.Value;
|
||||
}
|
||||
|
||||
public static bool operator >=(
|
||||
Ptr64 left,
|
||||
Ptr64 right
|
||||
)
|
||||
{
|
||||
return left.Value >= right.Value;
|
||||
}
|
||||
|
||||
public static Ptr64 operator >>(
|
||||
Ptr64 left,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
return new Ptr64(left.Value >> shift);
|
||||
}
|
||||
|
||||
public static implicit operator Ptr64(
|
||||
ulong value
|
||||
)
|
||||
{
|
||||
return new(value);
|
||||
}
|
||||
|
||||
public static implicit operator ulong(
|
||||
Ptr64 value
|
||||
)
|
||||
{
|
||||
return value.Value;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public readonly struct SPtr64 : IEquatable<SPtr64>, IComparable<SPtr64>, IFormattable
|
||||
{
|
||||
|
||||
public SPtr64(
|
||||
long value
|
||||
)
|
||||
{
|
||||
Value = value;
|
||||
}
|
||||
|
||||
public long Value { get; }
|
||||
|
||||
public int CompareTo(
|
||||
SPtr64 other
|
||||
)
|
||||
{
|
||||
return Value.CompareTo(other.Value);
|
||||
}
|
||||
|
||||
public bool Equals(
|
||||
SPtr64 other
|
||||
)
|
||||
{
|
||||
return Value == other.Value;
|
||||
}
|
||||
|
||||
public override bool Equals(
|
||||
object? obj
|
||||
)
|
||||
{
|
||||
return obj is SPtr64 other && Equals(other);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return Value.GetHashCode();
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return Value.ToString();
|
||||
}
|
||||
|
||||
public string ToString(
|
||||
string? format,
|
||||
IFormatProvider? formatProvider
|
||||
)
|
||||
{
|
||||
return Value.ToString(format, formatProvider);
|
||||
}
|
||||
|
||||
public static SPtr64 operator -(
|
||||
SPtr64 left,
|
||||
long right
|
||||
)
|
||||
{
|
||||
return new(left.Value - right);
|
||||
}
|
||||
|
||||
public static long operator -(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value - right.Value;
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value != right.Value;
|
||||
}
|
||||
|
||||
public static SPtr64 operator &(
|
||||
SPtr64 left,
|
||||
long mask
|
||||
)
|
||||
{
|
||||
return new(left.Value & mask);
|
||||
}
|
||||
|
||||
public static SPtr64 operator |(
|
||||
SPtr64 left,
|
||||
long mask
|
||||
)
|
||||
{
|
||||
return new(left.Value | mask);
|
||||
}
|
||||
|
||||
public static SPtr64 operator +(
|
||||
SPtr64 left,
|
||||
long right
|
||||
)
|
||||
{
|
||||
return new(left.Value + right);
|
||||
}
|
||||
|
||||
public static bool operator <(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value < right.Value;
|
||||
}
|
||||
|
||||
public static SPtr64 operator <<(
|
||||
SPtr64 left,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
return new SPtr64(left.Value << shift);
|
||||
}
|
||||
|
||||
public static bool operator <=(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value <= right.Value;
|
||||
}
|
||||
|
||||
public static bool operator ==(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value == right.Value;
|
||||
}
|
||||
|
||||
public static bool operator >(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value > right.Value;
|
||||
}
|
||||
|
||||
public static bool operator >=(
|
||||
SPtr64 left,
|
||||
SPtr64 right
|
||||
)
|
||||
{
|
||||
return left.Value >= right.Value;
|
||||
}
|
||||
|
||||
public static SPtr64 operator >>(
|
||||
SPtr64 left,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
return new SPtr64(left.Value >> shift);
|
||||
}
|
||||
|
||||
public static implicit operator long(
|
||||
SPtr64 value
|
||||
)
|
||||
{
|
||||
return value.Value;
|
||||
}
|
||||
|
||||
public static implicit operator SPtr64(
|
||||
long value
|
||||
)
|
||||
{
|
||||
return new(value);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#pragma warning disable CA2225
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocationHandle"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe struct SlabAllocationHandle<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
private readonly SlabAllocator<T> _allocator;
|
||||
|
||||
internal SlabAllocationHandle(
|
||||
SlabAllocator<T> allocator,
|
||||
bool locked
|
||||
)
|
||||
{
|
||||
_allocator = allocator;
|
||||
Locked = locked;
|
||||
}
|
||||
|
||||
public bool Locked { get; set; }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Locked)
|
||||
{
|
||||
_allocator.ExitBufferReading();
|
||||
Locked = false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocator.Chunked"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal sealed unsafe class ChunkedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
|
||||
{
|
||||
|
||||
private bool _disposed;
|
||||
|
||||
private BufferStack<nuint> _buffers;
|
||||
private BufferStack<nuint> _recycle;
|
||||
private readonly int _elementSize = Unsafe.SizeOf<T>();
|
||||
|
||||
private nuint _currentBuffer;
|
||||
private nuint _currentCapacity, _currentIndex;
|
||||
|
||||
public ChunkedSlabAllocator(
|
||||
nuint count
|
||||
)
|
||||
{
|
||||
_currentBuffer = (nuint)NativeMemory.Alloc(count * ((nuint)_elementSize));
|
||||
|
||||
_currentCapacity = count;
|
||||
int initCount = count switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)count
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _currentBuffer + ((nuint)(i * _elementSize));
|
||||
}
|
||||
_buffers = new(4);
|
||||
_buffers.Push(_currentBuffer);
|
||||
_recycle = BufferStackBuilder.Create(span);
|
||||
_currentCapacity = count;
|
||||
_currentIndex = (nuint)initCount;
|
||||
}
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycle.TryPop(out nuint handle)) {
|
||||
return handle;
|
||||
}
|
||||
|
||||
// nuint bufferHandle = _currentBuffer;
|
||||
nuint curIndex = _currentIndex;
|
||||
if (curIndex >= _currentCapacity)
|
||||
{
|
||||
nuint cap = _currentCapacity;
|
||||
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * ((nuint)_elementSize));
|
||||
_buffers.Push(newBuffer);
|
||||
_currentBuffer = newBuffer;
|
||||
_currentIndex = 0;
|
||||
}
|
||||
curIndex = _currentIndex;
|
||||
_currentIndex += 1;
|
||||
return _currentBuffer + (((nuint)_elementSize) * curIndex);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
|
||||
// NativeMemory.Free((void*)_currentBuffer);
|
||||
while (_buffers.TryPop(out nuint buffer))
|
||||
{
|
||||
NativeMemory.Free((void*)buffer);
|
||||
}
|
||||
_buffers.Dispose();
|
||||
_recycle.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
return new SlabAllocationHandle<T>(this, false);
|
||||
}
|
||||
|
||||
~ChunkedSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocator.ChunkedConcurrent"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal sealed unsafe class ConcurrentChunkedSlabAllocator<T> : SlabAllocator<T>
|
||||
where T : unmanaged
|
||||
{
|
||||
|
||||
private bool _disposed;
|
||||
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private ConcurrentBufferStack<nuint> _buffers;
|
||||
private ConcurrentBufferStack<nuint> _recycle;
|
||||
private readonly int _elementSize = Unsafe.SizeOf<T>();
|
||||
|
||||
private nuint _currentBuffer;
|
||||
private nuint _currentCapacity, _currentIndex;
|
||||
|
||||
public ConcurrentChunkedSlabAllocator(
|
||||
nuint count
|
||||
)
|
||||
{
|
||||
_currentBuffer = (nuint)NativeMemory.Alloc(count * (nuint)_elementSize);
|
||||
_currentCapacity = count;
|
||||
int initCount = count switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)count
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _currentBuffer + (nuint)(i * _elementSize);
|
||||
}
|
||||
_buffers = new(4);
|
||||
_recycle = BufferStackBuilder.CreateConcurrent(span);
|
||||
_currentCapacity = count;
|
||||
_currentIndex = (nuint)initCount;
|
||||
}
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycle.TryPop(out nuint handle)) {
|
||||
return handle;
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
nuint bufferHandle = Volatile.Read(ref _currentBuffer);
|
||||
nuint curIndex = Volatile.Read(ref _currentIndex);
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
if (curIndex >= _currentCapacity)
|
||||
{
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
nuint cap = Volatile.Read(ref _currentCapacity);
|
||||
nuint newBuffer = (nuint)NativeMemory.Alloc(cap * (nuint)_elementSize);
|
||||
_buffers.Push(bufferHandle);
|
||||
_ = Interlocked.Exchange(ref _currentBuffer, newBuffer);
|
||||
_ = Interlocked.Exchange(ref _currentIndex, 0);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
curIndex = Volatile.Read(ref _currentIndex);
|
||||
_ = Interlocked.Add(
|
||||
ref Unsafe.As<nuint, ulong>(ref _currentIndex), (ulong)_elementSize);
|
||||
return _currentBuffer + ((nuint)_elementSize * curIndex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
NativeMemory.Free((void*)_currentBuffer);
|
||||
while (_buffers.TryPop(out nuint buffer)) {
|
||||
NativeMemory.Free((void*)buffer);
|
||||
}
|
||||
_lock.Dispose();
|
||||
_buffers.Dispose();
|
||||
_recycle.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
return new SlabAllocationHandle<T>(this, false);
|
||||
}
|
||||
|
||||
~ConcurrentChunkedSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocator.Fixed"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using StgSharp.Threading;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
internal unsafe class FixedSlabAllocator<T> : SlabAllocator<T> where T : unmanaged
|
||||
{
|
||||
|
||||
private Action<nuint> _freeHandle;
|
||||
|
||||
private bool _disposed;
|
||||
private CapacityFixedStack<nuint> _recycle;
|
||||
private readonly int _elementSize = Unsafe.SizeOf<T>();
|
||||
|
||||
private nuint _buffer;
|
||||
|
||||
private nuint _Capacity, _currentIndex;
|
||||
|
||||
internal FixedSlabAllocator(
|
||||
nuint buffer,
|
||||
nuint count,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
_freeHandle = freeHandle;
|
||||
_buffer = buffer;
|
||||
|
||||
_Capacity = count;
|
||||
int initCount = count switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => (int)count
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = _buffer + ((nuint)(i * _elementSize));
|
||||
}
|
||||
_recycle = CapacityFixedStackBuilder.Create(span);
|
||||
_Capacity = count;
|
||||
_currentIndex = (nuint)initCount;
|
||||
}
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_recycle.TryPop(out nuint handle)) {
|
||||
return handle;
|
||||
}
|
||||
|
||||
// nuint bufferHandle = _currentBuffer;
|
||||
nuint curIndex = _currentIndex;
|
||||
if (curIndex >= _Capacity) {
|
||||
return 0;
|
||||
}
|
||||
curIndex = _currentIndex;
|
||||
_currentIndex += 1;
|
||||
return _buffer + (((nuint)_elementSize) * curIndex);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_freeHandle(_buffer);
|
||||
_recycle = null!;
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading() { }
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_recycle.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
return new SlabAllocationHandle<T>(this, false);
|
||||
}
|
||||
|
||||
~FixedSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocator.SequencialConcurrent"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public sealed unsafe class ConcurrentSequentialSlabAllocator<T> : SlabAllocator<T>
|
||||
where T : unmanaged
|
||||
{
|
||||
|
||||
private T* _buffer;
|
||||
|
||||
private bool _disposed;
|
||||
|
||||
private readonly BufferExpansionLock _lock = new();
|
||||
private readonly ConcurrentBufferStack<nuint> _stack;
|
||||
private readonly int _elementSize;
|
||||
private nuint _highestAllocated;
|
||||
private ulong _capacity;
|
||||
|
||||
public ConcurrentSequentialSlabAllocator(
|
||||
nuint initialCapacity
|
||||
)
|
||||
{
|
||||
_elementSize = Unsafe.SizeOf<T>();
|
||||
_capacity = initialCapacity;
|
||||
_highestAllocated = 0;
|
||||
|
||||
_buffer = (T*)NativeMemory.Alloc(((nuint)_capacity) * ((nuint)_elementSize));
|
||||
int initCount = _capacity switch
|
||||
{
|
||||
<= 4 => 4,
|
||||
> 64 => 64,
|
||||
_ => ((int)_capacity) / 4
|
||||
};
|
||||
Span<nuint> span = stackalloc nuint[initCount];
|
||||
for (int i = 0; i < initCount; i++) {
|
||||
span[i] = (nuint)i;
|
||||
}
|
||||
_stack = BufferStackBuilder.CreateConcurrent(span);
|
||||
_highestAllocated += (nuint)initCount;
|
||||
}
|
||||
|
||||
internal T* BasePointer => _buffer;
|
||||
|
||||
public override nuint Allocate()
|
||||
{
|
||||
if (_stack.TryPop(out nuint index)) {
|
||||
return index;
|
||||
}
|
||||
while (true)
|
||||
{
|
||||
index = Volatile.Read(ref _highestAllocated);
|
||||
_lock.EnterMetaDataRead();
|
||||
try
|
||||
{
|
||||
if (index >= _capacity)
|
||||
{
|
||||
_lock.ExitMetaDataRead();
|
||||
_lock.EnterExpansionProcess();
|
||||
try
|
||||
{
|
||||
_capacity = Volatile.Read(ref _capacity) * 2;
|
||||
nuint newSize = ((nuint)_capacity) * ((nuint)_elementSize);
|
||||
_lock.EnterBufferCopy();
|
||||
try
|
||||
{
|
||||
_buffer = (T*)NativeMemory.Realloc(_buffer, newSize);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitBufferCopy();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_lock.ExitExpansionProcess();
|
||||
}
|
||||
}
|
||||
_ = Interlocked.Increment(ref Unsafe.As<nuint, ulong>(ref _highestAllocated));
|
||||
return index;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (_lock.IsThreadReadingMetaData) {
|
||||
_lock.ExitMetaDataRead();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
if (_disposed) {
|
||||
return;
|
||||
}
|
||||
_disposed = true;
|
||||
_stack.Dispose();
|
||||
NativeMemory.Free(_buffer);
|
||||
_lock.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void EnterBufferReading()
|
||||
{
|
||||
_lock.EnterBufferRead();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override void ExitBufferReading()
|
||||
{
|
||||
_lock.ExitBufferRead();
|
||||
}
|
||||
|
||||
public override void Free(
|
||||
nuint index
|
||||
)
|
||||
{
|
||||
_stack.Push(index);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override SlabAllocationHandle<T> ReadAllocation()
|
||||
{
|
||||
EnterBufferReading();
|
||||
return new SlabAllocationHandle<T>(this, true);
|
||||
}
|
||||
|
||||
~ConcurrentSequentialSlabAllocator()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SlabAllocator"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using System;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public static class SlabAllocator
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new SLAB allocator instance with the specified configuration.
|
||||
/// </summary>
|
||||
/// <param name="count">
|
||||
/// Initial number of objects that can be allocated.
|
||||
/// </param>
|
||||
/// <param name="layout">
|
||||
/// Memory layout strategy for the SLAB allocator.
|
||||
/// </param>
|
||||
/// <param name="concurrentSupport">
|
||||
/// Whether to enable concurrent access support.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A new SLAB allocator instance.
|
||||
/// </returns>
|
||||
/// <exception cref="ArgumentOutOfRangeException">
|
||||
/// Thrown when an invalid layout is specified.
|
||||
/// </exception>
|
||||
/// <exception cref="NotSupportedException">
|
||||
/// Thrown when requesting unsupported configurations.
|
||||
/// </exception>
|
||||
/// <remarks>
|
||||
/// <para><strong> Important Threading Considerations: </strong></para> <para>While SLAB
|
||||
/// allocators themselves may be designed for single-threaded use, allocated objects are
|
||||
/// frequently shared across multiple threads in real-world applications. Due to the
|
||||
/// special implementation of expandable concurrent sequential SLAB allocators, they rely
|
||||
/// on internal expansion locks to ensure that memory buffer expansions (which involve
|
||||
/// copying the entire buffer to a new location) do not occur while other threads are
|
||||
/// reading from allocated regions. </para> <para><strong> Sequential Layout Limitations:
|
||||
/// </strong><br /> Sequential layout SLAB allocators cannot provide a single- threaded
|
||||
/// version because:<list type="bullet"><item><description> They use contiguous memory
|
||||
/// layout that requires buffer expansion when capacity is exceeded
|
||||
/// </description></item><item><description> Buffer expansion involves copying all
|
||||
/// existing data to a new memory location </description></item><item><description>Without
|
||||
/// proper synchronization, concurrent access to allocated objects during synchronization,
|
||||
/// concurrent access to allocated objects during expansion would cause memory
|
||||
/// corruption</description></item><item><description>The expansion lock mechanism is
|
||||
/// essential even if allocation itself is single- threaded
|
||||
/// </description></item></list></para> <para><strong> Chunked Layout: </strong><br />
|
||||
/// Chunked layout allocators can optionally provide single-threaded versions since they
|
||||
/// don't require buffer expansion and copying. Each chunk is allocated independently,
|
||||
/// avoiding the memory relocation issues present in sequential layouts. </para>
|
||||
/// </remarks>
|
||||
public static SlabAllocator<T> Create<T>(
|
||||
nuint count,
|
||||
SlabBufferLayout layout,
|
||||
bool concurrentSupport = true
|
||||
) where T : unmanaged
|
||||
{
|
||||
return layout switch
|
||||
{
|
||||
SlabBufferLayout.Sequential => concurrentSupport ?
|
||||
new ConcurrentSequentialSlabAllocator<T>(count) :
|
||||
throw (new NotSupportedException("Non-concurrent version of sequential layout SLAB is not supported due to expansion lock requirements for memory safety during buffer expansion.")),
|
||||
SlabBufferLayout.Chunked => concurrentSupport ?
|
||||
new ConcurrentChunkedSlabAllocator<T>(count) :
|
||||
new ChunkedSlabAllocator<T>(count),
|
||||
_ => throw (new ArgumentOutOfRangeException(nameof(layout), layout, null))
|
||||
};
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static SlabAllocator<T> FromBuffer<T>(
|
||||
nuint buffer,
|
||||
nuint count,
|
||||
Action<nuint> freeHandle
|
||||
) where T : unmanaged
|
||||
{
|
||||
return new FixedSlabAllocator<T>(buffer, count, freeHandle);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public abstract class SlabAllocator<T> : IDisposable where T : unmanaged
|
||||
{
|
||||
|
||||
public abstract nuint Allocate();
|
||||
|
||||
public abstract void Dispose();
|
||||
|
||||
public abstract void EnterBufferReading();
|
||||
|
||||
public abstract void ExitBufferReading();
|
||||
|
||||
public abstract void Free(
|
||||
nuint index
|
||||
);
|
||||
|
||||
public abstract SlabAllocationHandle<T> ReadAllocation();
|
||||
|
||||
}
|
||||
|
||||
public enum SlabBufferLayout
|
||||
{
|
||||
|
||||
Sequential,
|
||||
Chunked
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,669 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TwoLayerSegregatedFitAllocator"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.ComponentModel;
|
||||
using System.Drawing;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml;
|
||||
using static StgSharp.HighPerformance.Memory.TwoLayerSegregatedFitAllocator;
|
||||
|
||||
namespace StgSharp.HighPerformance.Memory
|
||||
{
|
||||
public unsafe class TwoLayerSegregatedFitAllocator : IDisposable
|
||||
{
|
||||
|
||||
private const ulong Mask48Bits = 0x0000FFFFFFFFFFFF;
|
||||
|
||||
private readonly int[] _bucket;
|
||||
private readonly Metadata* _spareMemory;
|
||||
private readonly Action<nuint> _freeHandle;
|
||||
private bool disposedValue;
|
||||
private readonly int _align;
|
||||
private readonly int _maxLevel;
|
||||
|
||||
private int _metadataEnd;
|
||||
private readonly Ptr64 _basePtr;
|
||||
private readonly ulong _maxAllocSize;
|
||||
private readonly ulong _totalSize;
|
||||
|
||||
internal TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
ulong maxBlockSize,
|
||||
Func<nuint, int, nuint> allocator,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
: this(byteSize, align, maxBlockSize, (void*)allocator(byteSize, align), freeHandle) { }
|
||||
|
||||
internal TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
ulong maxBlockSize,
|
||||
void* externalPointer,
|
||||
Action<nuint> freeHandle
|
||||
)
|
||||
{
|
||||
_totalSize = byteSize;
|
||||
_basePtr = (Ptr64)(ulong)externalPointer;
|
||||
_align = align;
|
||||
_maxAllocSize = ulong.Max(64, (~64UL) & (ulong.Min(maxBlockSize + 63, Mask48Bits)));
|
||||
_maxLevel = int.MaxValue;
|
||||
_maxLevel = GetLevel(_maxAllocSize);
|
||||
_bucket = new int[GetLevel(_maxAllocSize) + 2];
|
||||
_spareMemory = &((Metadata*)(ulong)(_basePtr + _totalSize))[-1];
|
||||
_freeHandle = freeHandle ?? (static_ => { });
|
||||
_metadataEnd = 0;
|
||||
|
||||
// init spare memory node
|
||||
|
||||
_spareMemory->Position = _basePtr;
|
||||
_spareMemory->SizeForSpare = _totalSize - Metadata.SizeOf;
|
||||
_spareMemory->NextLevel = 0;
|
||||
_spareMemory->NextNear = 0;
|
||||
_spareMemory->PreviousLevel = 0;
|
||||
_spareMemory->PreviousNear = 0;
|
||||
}
|
||||
|
||||
public TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize
|
||||
)
|
||||
: this(byteSize, 16, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
uint maxBlockSize
|
||||
)
|
||||
: this(byteSize, 16, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align
|
||||
)
|
||||
: this(byteSize, align, 32U * 1024U * 1024U, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public TwoLayerSegregatedFitAllocator(
|
||||
nuint byteSize,
|
||||
int align,
|
||||
nuint maxBlockSize
|
||||
)
|
||||
: this(byteSize, align, maxBlockSize, NativeMemory.AlignedAlloc(byteSize, 16),
|
||||
ptr => NativeMemory.AlignedFree((void*)ptr)) { }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(
|
||||
bool disposing
|
||||
)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing) {
|
||||
_freeHandle((nuint)_basePtr);
|
||||
}
|
||||
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetLevel(
|
||||
ulong size
|
||||
)
|
||||
{
|
||||
int origin = 57 - BitOperations.LeadingZeroCount(size);
|
||||
return int.Min(origin, _maxLevel);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Put a metadata node back to free list and waiting for next allocation.
|
||||
/// </summary>
|
||||
/// <param name="node">
|
||||
///
|
||||
/// </param>
|
||||
private void RecycleNode(
|
||||
Metadata* node
|
||||
)
|
||||
{
|
||||
int nodeIdx = (int)(_spareMemory - node);
|
||||
int level = GetLevel(node->Size);
|
||||
int nextLevelIdx = node->NextLevel;
|
||||
int prevLevelIdx = node->PreviousLevel;
|
||||
RemoveFromBucket(level, node, nodeIdx);
|
||||
|
||||
// remove from position chain
|
||||
int prevNearIdx = node->PreviousNear;
|
||||
int nextNearIdx = node->NextNear;
|
||||
Metadata* prevNear = &_spareMemory[-prevNearIdx];
|
||||
prevNear->NextNear = nextNearIdx;
|
||||
if (nextNearIdx != 0)
|
||||
{
|
||||
Metadata* nextNear = &_spareMemory[-nextNearIdx];
|
||||
nextNear->PreviousNear = prevNearIdx;
|
||||
} else
|
||||
{
|
||||
_spareMemory->PreviousNear = prevNearIdx;
|
||||
}
|
||||
node->PreviousNear = 0;
|
||||
node->NextNear = 0;
|
||||
|
||||
node->Position = 0;
|
||||
node->SizeForSpare = 0;
|
||||
node->State = AllocationState.SpareOrCollect;
|
||||
node->PreviousLevel = 0;
|
||||
|
||||
int nextIdx = _spareMemory->NextLevel;
|
||||
if (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[-nextIdx];
|
||||
next->PreviousLevel = nodeIdx;
|
||||
}
|
||||
node->NextLevel = nextIdx;
|
||||
_spareMemory->NextLevel = nodeIdx;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void RemoveFromBucket(
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
RemoveFromBucket(GetLevel(node->Size), node, nodeIdx);
|
||||
}
|
||||
|
||||
private void RemoveFromBucket(
|
||||
int level,
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
int currentNextLevelIdx = node->NextLevel;
|
||||
int currentPrevLevelIdx = node->PreviousLevel;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
} else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[-currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[-currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = currentPrevLevelIdx;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
// Assuming node to remove is head f bucket
|
||||
private void RemoveFromBucketHead(
|
||||
int level,
|
||||
Metadata* node,
|
||||
int nodeIdx
|
||||
)
|
||||
{
|
||||
int currentLevel = level;
|
||||
int currentNextLevelIdx = node->NextLevel;
|
||||
|
||||
// int currentPrevLevelIdx = 0;
|
||||
|
||||
if (_bucket[currentLevel] == nodeIdx)
|
||||
{
|
||||
_bucket[currentLevel] = currentNextLevelIdx;
|
||||
}
|
||||
/*
|
||||
else if (currentPrevLevelIdx != 0)
|
||||
{
|
||||
Metadata* prevLevel = &_spareMemory[-currentPrevLevelIdx];
|
||||
prevLevel->NextLevel = currentNextLevelIdx;
|
||||
}
|
||||
/**/
|
||||
if (currentNextLevelIdx != 0)
|
||||
{
|
||||
Metadata* nextLevel = &_spareMemory[-currentNextLevelIdx];
|
||||
nextLevel->PreviousLevel = 0;
|
||||
}
|
||||
node->PreviousLevel = 0;
|
||||
node->NextLevel = 0;
|
||||
}
|
||||
|
||||
private bool TryAllocNewChunk(
|
||||
ulong size,
|
||||
out Metadata* meta
|
||||
)
|
||||
{
|
||||
int idx = _spareMemory->NextLevel;
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
if (idx != 0)
|
||||
{
|
||||
if (spareSize < size)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
meta = &_spareMemory[-idx];
|
||||
int nextSpareIdx = meta->NextLevel;
|
||||
if (nextSpareIdx != 0)
|
||||
{
|
||||
Metadata* nextSpare = &_spareMemory[-nextSpareIdx];
|
||||
nextSpare->PreviousLevel = 0;
|
||||
}
|
||||
_spareMemory->NextLevel = nextSpareIdx;
|
||||
} else
|
||||
{
|
||||
if (spareSize < Metadata.SizeOf + size)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
_metadataEnd++;
|
||||
idx = _metadataEnd;
|
||||
meta = &_spareMemory[-idx];
|
||||
spareSize -= Metadata.SizeOf;
|
||||
}
|
||||
spareSize -= size;
|
||||
meta->Size = size;
|
||||
meta->Position = _spareMemory->Position;
|
||||
meta->State = AllocationState.Free;
|
||||
meta->PreviousLevel = 0;
|
||||
meta->NextLevel = 0;
|
||||
_spareMemory->Size = spareSize;
|
||||
_spareMemory->Position += size;
|
||||
|
||||
// insert the new chunk to position chain
|
||||
int nearIdx = _spareMemory->PreviousNear;
|
||||
Metadata* near = &_spareMemory[-nearIdx];
|
||||
near->NextNear = idx;
|
||||
meta->PreviousNear = nearIdx;
|
||||
_spareMemory->PreviousNear = idx;
|
||||
meta->NextNear = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryGetNewMetadata(
|
||||
out Metadata* meta
|
||||
)
|
||||
{
|
||||
ulong spareSize = _spareMemory->Size;
|
||||
if (spareSize < Metadata.SizeOf)
|
||||
{
|
||||
meta = null;
|
||||
return false;
|
||||
}
|
||||
_metadataEnd++;
|
||||
meta = &_spareMemory[-_metadataEnd];
|
||||
spareSize -= Metadata.SizeOf;
|
||||
_spareMemory->Size = spareSize;
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool TryTakeBucketEntry(
|
||||
ulong minimumSize,
|
||||
out Metadata* entry
|
||||
)
|
||||
{
|
||||
int level = GetLevel(minimumSize);
|
||||
|
||||
// first try to find in the target level
|
||||
{
|
||||
int idx = _bucket[level];
|
||||
while (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[-idx];
|
||||
if (candidate->Size >= minimumSize)
|
||||
{
|
||||
RemoveFromBucket(level, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
idx = candidate->NextLevel;
|
||||
}
|
||||
}
|
||||
for (int i = level + 1; i < _bucket.Length; i++)
|
||||
{
|
||||
int idx = _bucket[i];
|
||||
if (idx != 0)
|
||||
{
|
||||
Metadata* candidate = &_spareMemory[-idx];
|
||||
RemoveFromBucketHead(i, candidate, idx);
|
||||
entry = candidate;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
entry = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit)]
|
||||
public readonly struct Handle
|
||||
{
|
||||
|
||||
[FieldOffset(0)] internal readonly Metadata* _meta;
|
||||
[FieldOffset(8)] public readonly Ptr64 Address;
|
||||
[FieldOffset(16)] public readonly Ptr64 Size;
|
||||
|
||||
internal Handle(
|
||||
Metadata* meta,
|
||||
Ptr64 baseAddress
|
||||
)
|
||||
{
|
||||
_meta = meta;
|
||||
Address = (ulong)meta->Position;
|
||||
Size = meta->Size;
|
||||
}
|
||||
|
||||
[Obsolete("You should not create instance from default. Alloc from allocator please")]
|
||||
public Handle() { }
|
||||
|
||||
public ref byte this[
|
||||
int offset
|
||||
] => ref ((byte*)(ulong)Address)[offset];
|
||||
|
||||
public Span<byte> AsSpan(
|
||||
int range = -1
|
||||
)
|
||||
{
|
||||
return new Span<byte>((byte*)(ulong)Address, range < 0 ? (int)(ulong)Size : range);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal enum AllocationState : byte
|
||||
{
|
||||
|
||||
SpareOrCollect = 0,
|
||||
Free = 1,
|
||||
Alloc = 2,
|
||||
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Explicit, Size = 32)]
|
||||
internal struct Metadata
|
||||
{
|
||||
|
||||
[FieldOffset(16)] private ulong PositionMask;
|
||||
[FieldOffset(24)] private ulong SizeMask;
|
||||
|
||||
[FieldOffset(30)] internal AllocationState State;
|
||||
[FieldOffset(31)] internal sbyte Level;
|
||||
|
||||
[FieldOffset(0)] public int NextLevel;
|
||||
[FieldOffset(4)] public int NextNear;
|
||||
[FieldOffset(8)] public int PreviousLevel;
|
||||
[FieldOffset(12)] public int PreviousNear;
|
||||
|
||||
public Ptr64 Position
|
||||
{
|
||||
readonly get => PositionMask;
|
||||
set => PositionMask = value;
|
||||
}
|
||||
|
||||
public Ptr64 Size
|
||||
{
|
||||
readonly get => (Ptr64)(SizeMask & Mask48Bits);
|
||||
set => SizeMask = (SizeMask & ~Mask48Bits) | ((ulong)value & Mask48Bits);
|
||||
}
|
||||
|
||||
public Ptr64 SizeForSpare
|
||||
{
|
||||
readonly get => SizeMask;
|
||||
set => SizeMask = value;
|
||||
}
|
||||
|
||||
public static uint SizeOf { get; } = (uint)Unsafe.SizeOf<Metadata>();
|
||||
|
||||
}
|
||||
|
||||
#region alloc and free
|
||||
|
||||
public Handle Alloc(
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
|
||||
if (allocSize > _maxAllocSize) {
|
||||
throw new OverflowException("Requested allocation size exceeds the maximum block size.");
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry)) {
|
||||
throw new OverflowException("No available ");
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
if (entrySize < allocSize) {
|
||||
throw new OverflowException("Bucket returned an undersized block.");
|
||||
}
|
||||
ulong remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
}
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[-afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->Position - (ulong)_basePtr > _totalSize) {
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
return new Handle(entry, _basePtr);
|
||||
}
|
||||
|
||||
public void Collect()
|
||||
{
|
||||
int currentIdx = _spareMemory->NextNear;
|
||||
do
|
||||
{
|
||||
if (currentIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* current = &_spareMemory[-currentIdx];
|
||||
if (current->State == AllocationState.Free)
|
||||
{
|
||||
// scan next
|
||||
Ptr64 size = 0;
|
||||
int nextIdx = current->NextNear;
|
||||
while (nextIdx != 0)
|
||||
{
|
||||
Metadata* next = &_spareMemory[-nextIdx];
|
||||
if (next->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += next->Size;
|
||||
RecycleNode(next);
|
||||
nextIdx = current->NextNear;
|
||||
}
|
||||
if (size > 0)
|
||||
{
|
||||
current->Size += size;
|
||||
RemoveFromBucket(current, currentIdx);
|
||||
int newLevel = GetLevel(current->Size);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = currentIdx;
|
||||
}
|
||||
current->NextLevel = bucketHeadIdx;
|
||||
current->PreviousLevel = 0;
|
||||
_bucket[newLevel] = currentIdx;
|
||||
}
|
||||
}
|
||||
currentIdx = current->NextNear;
|
||||
} while (true);
|
||||
}
|
||||
|
||||
public void Free(
|
||||
Handle handle,
|
||||
int collectScanWindow = 4
|
||||
)
|
||||
{
|
||||
Metadata* head = handle._meta;
|
||||
int headIdx = (int)(_spareMemory - head);
|
||||
Ptr64 size = handle.Size;
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// forward scan, previous direction
|
||||
int forwardIdx = head->PreviousNear;
|
||||
if (forwardIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[-forwardIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
head->Position = prev->Position;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
for (int i = 0; i < collectScanWindow; i++)
|
||||
{
|
||||
// backward scan, next direction
|
||||
int nextIdx = head->NextNear;
|
||||
if (nextIdx == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
Metadata* prev = &_spareMemory[-nextIdx];
|
||||
if (prev->State != AllocationState.Free)
|
||||
{
|
||||
break;
|
||||
}
|
||||
size += prev->Size;
|
||||
RecycleNode(prev);
|
||||
}
|
||||
|
||||
// recycle to level;
|
||||
head->Size = size;
|
||||
head->State = AllocationState.Free;
|
||||
head->PreviousLevel = 0;
|
||||
int level = GetLevel(head->Size);
|
||||
int bucketHeadIdx = _bucket[level];
|
||||
head->NextLevel = 0;
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
|
||||
bucketHead->PreviousLevel = headIdx;
|
||||
head->NextLevel = bucketHeadIdx;
|
||||
}
|
||||
_bucket[level] = headIdx;
|
||||
return;
|
||||
}
|
||||
|
||||
public bool TryAlloc(
|
||||
nuint size,
|
||||
out Handle h
|
||||
)
|
||||
{
|
||||
ulong allocSize = ulong.Max(64, (~63UL) & ((ulong)size + 63));
|
||||
if (allocSize > _maxAllocSize)
|
||||
{
|
||||
h = default;
|
||||
return false;
|
||||
}
|
||||
if (!TryTakeBucketEntry(allocSize, out Metadata* entry) &&
|
||||
!TryAllocNewChunk(allocSize, out entry))
|
||||
{
|
||||
h = default;
|
||||
return false;
|
||||
}
|
||||
int entryIdx = (int)(_spareMemory - entry);
|
||||
ulong entrySize = entry->Size;
|
||||
if (entrySize < allocSize)
|
||||
{
|
||||
h = default;
|
||||
return false;
|
||||
}
|
||||
ulong remain = entrySize - allocSize;
|
||||
|
||||
// slice tail
|
||||
if (remain > (entrySize >> 7) && TryGetNewMetadata(out Metadata* newEntry))
|
||||
{
|
||||
int newEntryIdx = (int)(_spareMemory - newEntry);
|
||||
newEntry->Position = entry->Position + allocSize;
|
||||
newEntry->Size = remain;
|
||||
newEntry->State = AllocationState.Free;
|
||||
int newLevel = GetLevel(remain);
|
||||
int bucketHeadIdx = _bucket[newLevel];
|
||||
if (bucketHeadIdx != 0)
|
||||
{
|
||||
// if there is already an entry in the bucket, link it to the new entry
|
||||
Metadata* bucketHead = &_spareMemory[-bucketHeadIdx];
|
||||
newEntry->NextLevel = bucketHeadIdx;
|
||||
bucketHead->PreviousLevel = newEntryIdx;
|
||||
} else
|
||||
{
|
||||
newEntry->NextLevel = 0;
|
||||
}
|
||||
newEntry->PreviousLevel = 0;
|
||||
_bucket[newLevel] = newEntryIdx;
|
||||
|
||||
// push to position chain
|
||||
int afterIdx = entry->NextNear;
|
||||
Metadata* after = &_spareMemory[-afterIdx];
|
||||
newEntry->NextNear = afterIdx;
|
||||
after->PreviousNear = newEntryIdx;
|
||||
entry->NextNear = newEntryIdx;
|
||||
newEntry->PreviousNear = entryIdx;
|
||||
}
|
||||
entry->State = AllocationState.Alloc;
|
||||
if (entry->Position - (ulong)_basePtr > _totalSize)
|
||||
{
|
||||
h = default;
|
||||
throw new OverflowException("Allocated memory address exceeds the maximum block size.");
|
||||
}
|
||||
h = new Handle(entry, _basePtr);
|
||||
return true;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="IRegisterType"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
/// <summary>
|
||||
/// Abstraction for a wide SIMD register or equivalent packed numeric storage. Provides raw
|
||||
/// reference access, broadcast initialization and element level access. Typical
|
||||
/// implementations map to 128/256/512-bit hardware registers or software emulated blocks.
|
||||
/// </summary>
|
||||
|
||||
public interface IRegisterPresentation
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Gets a reference to the first element of the register reinterpreted as type <typeparamref
|
||||
/// name="T" />.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
|
||||
/// </typeparam>
|
||||
/// <returns>
|
||||
/// Writable reference to element 0 (lowest lane / lowest bits).
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Intended for treating the register as a contiguous block of elements. The returned reference
|
||||
/// must remain valid for the lifetime of the register and be identical to <see
|
||||
/// cref="Member{T}(int)" /> with index 0. Caller must choose a type layout compatible with the
|
||||
/// actual register representation.
|
||||
/// </remarks>
|
||||
ref T AsRef<T>() where T : unmanaged, INumber<T>;
|
||||
|
||||
/// <summary>
|
||||
/// Fills all elements (lanes) of the register with the provided scalar <paramref name="value" />.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
|
||||
/// </typeparam>
|
||||
/// <param name="value">
|
||||
/// Scalar value to replicate to every element.
|
||||
/// </param>
|
||||
/// <remarks>
|
||||
/// Should map to a single broadcast/shuffle instruction where possible. After completion all
|
||||
/// lanes read back the same value.
|
||||
/// </remarks>
|
||||
void BroadCastFrom<T>(
|
||||
T value
|
||||
) where T : unmanaged, INumber<T>;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a writable reference to the element at the specified <paramref name="index" />.
|
||||
/// </summary>
|
||||
/// <typeparam name="T">
|
||||
/// Unmanaged numeric element type implementing <see cref="INumber{T}" />.
|
||||
/// </typeparam>
|
||||
/// <param name="index">
|
||||
/// Zero-based lane index (range defined by implementation).
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// Writable reference to the lane.
|
||||
/// </returns>
|
||||
/// <remarks>
|
||||
/// Enables scalar lane edits without unpacking the entire register. Implementations may omit
|
||||
/// bounds checks for performance. Caller is responsible for passing a valid index.
|
||||
/// </remarks>
|
||||
ref T Member<T>(
|
||||
int index
|
||||
) where T : unmanaged, INumber<T>;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="LogicalCore"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public readonly struct LogicalCore
|
||||
{
|
||||
|
||||
public readonly uint GlobalCpuId;
|
||||
public readonly uint NumaGroupId;
|
||||
|
||||
/// <summary>
|
||||
/// Stays 0 on linux.
|
||||
/// </summary>
|
||||
public readonly uint NumaNodeId;
|
||||
|
||||
internal LogicalCore(
|
||||
uint globalCpuId,
|
||||
uint numaGroupId,
|
||||
uint numaNodeId
|
||||
)
|
||||
{
|
||||
GlobalCpuId = globalCpuId;
|
||||
NumaGroupId = numaGroupId;
|
||||
NumaNodeId = numaNodeId;
|
||||
}
|
||||
|
||||
[Obsolete(
|
||||
"Do not manually create instances of LogicalCore. Use NumaNode to obtain LogicalCore instances.",
|
||||
true)]
|
||||
public LogicalCore() { }
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="M128"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Runtime.Intrinsics.X86;
|
||||
|
||||
using System.Text;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
[StructLayout(LayoutKind.Explicit, Pack = 16)]
|
||||
public unsafe struct M128 : IRegisterPresentation
|
||||
{
|
||||
|
||||
[FieldOffset(0)] internal fixed byte Buffer[16];
|
||||
[FieldOffset(0)] internal Vector128<float> Vec;
|
||||
|
||||
internal M128(
|
||||
Vector4 v
|
||||
)
|
||||
{
|
||||
Unsafe.SkipInit(out this);
|
||||
Unsafe.SkipInit(out Vec);
|
||||
Vec = v.AsVector128();
|
||||
}
|
||||
|
||||
internal M128(
|
||||
Vector128<float> v
|
||||
)
|
||||
{
|
||||
Unsafe.SkipInit(out this);
|
||||
Vec = v;
|
||||
}
|
||||
|
||||
public ref T AsRef<T>() where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref Buffer[0]);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public readonly Vector4 AsVector4()
|
||||
{
|
||||
return Vec.AsVector4();
|
||||
}
|
||||
|
||||
public void BroadCastFrom<T>(
|
||||
T value
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
Vec = Vector128.Create<T>(value).AsSingle();
|
||||
}
|
||||
|
||||
public override readonly bool Equals(
|
||||
[NotNullWhen(true)] object obj
|
||||
)
|
||||
{
|
||||
return obj is M128 other && this == other;
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return Vec.GetHashCode();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ref T Member<T>(
|
||||
int index
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.Add(ref AsRef<T>(), index);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
public string ToString(
|
||||
int sample
|
||||
)
|
||||
{
|
||||
string str = string.Empty;
|
||||
for (int i = 15; i >= 0; i--) {
|
||||
str += $"{Convert.ToString(Buffer[i], sample)} ";
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool operator !=(
|
||||
M128 left,
|
||||
M128 right
|
||||
)
|
||||
{
|
||||
return left.Vec != right.Vec;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static bool operator ==(
|
||||
M128 left,
|
||||
M128 right
|
||||
)
|
||||
{
|
||||
return left.Vec == right.Vec;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="M256"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
[StructLayout(LayoutKind.Explicit, Pack = 16)]
|
||||
public unsafe struct M256 : IRegisterPresentation
|
||||
{
|
||||
|
||||
[FieldOffset(0)] internal fixed byte Buffer[32];
|
||||
[FieldOffset(0)] internal Vector256<float> Vec;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ref T AsRef<T>() where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref Buffer[0]);
|
||||
}
|
||||
|
||||
public void BroadCastFrom<T>(
|
||||
T value
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
Vec = Vector256.Create(value).AsSingle();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ref T Member<T>(
|
||||
int index
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.Add(ref AsRef<T>(), index);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="M512"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
[StructLayout(LayoutKind.Explicit, Size = 64, Pack = 64)]
|
||||
public unsafe struct M512 : IRegisterPresentation
|
||||
{
|
||||
|
||||
[FieldOffset(0)] private fixed byte _buffer[64];
|
||||
[FieldOffset(0)] private Vector512<byte> Vec;
|
||||
|
||||
public M512()
|
||||
{
|
||||
Unsafe.SkipInit(out this);
|
||||
Unsafe.SkipInit(out Vec);
|
||||
}
|
||||
|
||||
public ref T AsRef<T>() where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref _buffer[0]);
|
||||
}
|
||||
|
||||
public void BroadCastFrom<T>(
|
||||
T value
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
Vec = Vector512.Create(value).AsByte();
|
||||
}
|
||||
|
||||
public ref T Member<T>(
|
||||
int index
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref _buffer[index]);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="M64"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.Intrinsics;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
[StructLayout(LayoutKind.Explicit, Pack = 8)]
|
||||
public unsafe struct M64 : IRegisterPresentation
|
||||
{
|
||||
|
||||
[FieldOffset(0)] private fixed byte buffer[8];
|
||||
[FieldOffset(0)] private ulong value;
|
||||
[FieldOffset(0)] private Vector64<float> vec;
|
||||
|
||||
public M64()
|
||||
{
|
||||
Unsafe.SkipInit(out this);
|
||||
Unsafe.SkipInit(out vec);
|
||||
Unsafe.SkipInit(out value);
|
||||
}
|
||||
|
||||
public ref T AsRef<T>() where T : unmanaged,INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref buffer[0]);
|
||||
}
|
||||
|
||||
public void BroadCastFrom<T>(
|
||||
T value
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
vec = Vector64.Create(value).AsSingle();
|
||||
}
|
||||
|
||||
public override bool Equals(
|
||||
object obj
|
||||
)
|
||||
{
|
||||
return obj is M64 m64 && this == m64;
|
||||
}
|
||||
|
||||
public override readonly int GetHashCode()
|
||||
{
|
||||
return value.GetHashCode();
|
||||
}
|
||||
|
||||
public ulong GetLowerBits(
|
||||
int bitCount
|
||||
)
|
||||
{
|
||||
return value & ((1UL << bitCount) - 1);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public ref T Member<T>(
|
||||
int index
|
||||
) where T : unmanaged, INumber<T>
|
||||
{
|
||||
return ref Unsafe.As<byte, T>(ref buffer[index * sizeof(T)]);
|
||||
}
|
||||
|
||||
public void SetLowerBits(
|
||||
int bitCount,
|
||||
ulong bits
|
||||
)
|
||||
{
|
||||
value = (value & (~(1UL << bitCount))) | (bits & (1UL << bitCount));
|
||||
}
|
||||
|
||||
public static bool operator !=(
|
||||
M64 left,
|
||||
M64 right
|
||||
)
|
||||
{
|
||||
return !(left == right);
|
||||
}
|
||||
|
||||
public static M64 operator <<(
|
||||
M64 value,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
M64 result = new M64();
|
||||
result.value = value.value << shift;
|
||||
return result;
|
||||
}
|
||||
|
||||
public static bool operator ==(
|
||||
M64 left,
|
||||
M64 right
|
||||
)
|
||||
{
|
||||
return left.value == right.value;
|
||||
}
|
||||
public static M64 operator >>(
|
||||
M64 value,
|
||||
int shift
|
||||
)
|
||||
{
|
||||
return new()
|
||||
{
|
||||
value = value.value << shift
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Numa"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Contracts;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public static partial class Numa
|
||||
{
|
||||
|
||||
private static NumaNode[] _numaNodes;
|
||||
|
||||
public static int NumaNodeCount { get; private set; }
|
||||
|
||||
public static ReadOnlySpan<NumaNode> AllNumaNode => _numaNodes;
|
||||
|
||||
public static unsafe void* Alloc(
|
||||
int numaId,
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
|
||||
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
|
||||
}
|
||||
return NumaAllocNative(size, numaId);
|
||||
}
|
||||
|
||||
public static T[][] CreateForeachThread<T>(
|
||||
Func<LogicalCore, T> factory
|
||||
)
|
||||
{
|
||||
if (factory is null) {
|
||||
return [];
|
||||
}
|
||||
T[][] arr = new T[_numaNodes.Length][];
|
||||
for (int i = 0; i < _numaNodes.Length; i++)
|
||||
{
|
||||
NumaNode node = _numaNodes[i];
|
||||
T[] values = new T[node.LogicalCoreCount];
|
||||
ReadOnlySpan<LogicalCore> cores = node.LogicalCores;
|
||||
for (int j = 0; j < cores.Length; j++) {
|
||||
values[j] = factory(cores[j]);
|
||||
}
|
||||
arr[i] = values;
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
public static unsafe void Free(
|
||||
void* address,
|
||||
int numaId,
|
||||
nuint size
|
||||
)
|
||||
{
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
if ((numaId < 0) || (numaId >= _numaNodes.Length)) {
|
||||
throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID.");
|
||||
}
|
||||
NumaFreeNative(address, size, numaId);
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_each")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaEach(
|
||||
uint numaId,
|
||||
Span<ulong> CpuIndexInNuma,
|
||||
Span<uint> globalCpuIdInNuma
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_overall")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
public static partial int GetNumaOverall(
|
||||
ref int count,
|
||||
Span<int> coreCountInNuma
|
||||
);
|
||||
|
||||
public static L4[] RequestBufferForEachNuma()
|
||||
{
|
||||
if (_numaNodes.Length == 0) {
|
||||
throw new InvalidOperationException("Numa info has not been loaded.");
|
||||
}
|
||||
L4[] buffers = new L4[NumaNodeCount];
|
||||
for (int i = 0; i < NumaNodeCount; i++)
|
||||
{
|
||||
L4 buffer = new L4(i);
|
||||
buffers[i] = buffer;
|
||||
}
|
||||
return buffers;
|
||||
}
|
||||
|
||||
internal static void Initialize()
|
||||
{
|
||||
Span<int> coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1];
|
||||
int numaNodeCount = 0;
|
||||
if (0 == GetNumaOverall(ref numaNodeCount, coreCountInNuma)) {
|
||||
throw new InvalidOperationException("Failed to get NUMA overall information.");
|
||||
}
|
||||
_numaNodes = new NumaNode[numaNodeCount];
|
||||
for (uint i = 0; i < numaNodeCount; i++) {
|
||||
_numaNodes[i] = InitNumaNode(i, coreCountInNuma[(int)i]);
|
||||
}
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
internal static partial void SetThreadAffinity(
|
||||
int coreId
|
||||
);
|
||||
|
||||
[MethodImpl(MethodImplOptions.NoInlining)]
|
||||
private static NumaNode InitNumaNode(
|
||||
uint numaId,
|
||||
int count
|
||||
)
|
||||
{
|
||||
Span<uint> globalIndex = stackalloc uint[count];
|
||||
_ = GetNumaEach(numaId, null, globalIndex);
|
||||
return new(numaId, count, globalIndex);
|
||||
}
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_alloc")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void* NumaAllocNative(
|
||||
nuint size,
|
||||
int numaId
|
||||
);
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_numa_free")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static unsafe partial void NumaFreeNative(
|
||||
void* ptr,
|
||||
nuint size,
|
||||
int numa_id
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="NumaNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.HighPerformance.ProcessorAbstraction
|
||||
{
|
||||
public class NumaNode
|
||||
{
|
||||
|
||||
private LogicalCore[] _logicalCores;
|
||||
|
||||
internal NumaNode(
|
||||
uint numaId,
|
||||
int count,
|
||||
Span<uint> logicalCoreIds
|
||||
|
||||
)
|
||||
{
|
||||
_logicalCores = new LogicalCore[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
_logicalCores[i] = new LogicalCore(1, numaId, logicalCoreIds[i]);
|
||||
}
|
||||
}
|
||||
|
||||
public int LogicalCoreCount => _logicalCores.Length;
|
||||
|
||||
public ReadOnlySpan<LogicalCore> LogicalCores => _logicalCores;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="DefaultLog"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Internal
|
||||
{
|
||||
internal static unsafe partial class DefaultLog
|
||||
{
|
||||
|
||||
private const long MaxLogFileSize = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
private static LogType _currentLogLevel = LogType.Info;
|
||||
|
||||
internal static readonly GCHandle ssc_logHandle =
|
||||
GCHandle.Alloc(new byte[512], GCHandleType.Pinned);
|
||||
|
||||
internal static SemaphoreSlim logSyncSemaphore = new(1, 1);
|
||||
|
||||
internal static void InternalAppendLog(
|
||||
string log
|
||||
)
|
||||
{
|
||||
if (!File.Exists(Native.LogPath))
|
||||
{
|
||||
FileStream stream = File.Create(Native.LogPath);
|
||||
stream.Close();
|
||||
}
|
||||
|
||||
using (StreamWriter logStream = File.AppendText(Native.LogPath)) {
|
||||
logStream.WriteLine(log);
|
||||
}
|
||||
}
|
||||
|
||||
internal static void InternalWriteLog(
|
||||
string logLine,
|
||||
LogType logType
|
||||
)
|
||||
{
|
||||
if (logType < _currentLogLevel) {
|
||||
return;
|
||||
}
|
||||
|
||||
logSyncSemaphore.Wait();
|
||||
try
|
||||
{
|
||||
RotateLogFile();
|
||||
|
||||
if (!File.Exists(Native.LogPath))
|
||||
{
|
||||
FileStream stream = File.Create(Native.LogPath);
|
||||
stream.Close();
|
||||
}
|
||||
|
||||
using (StreamWriter logStream = File.AppendText(Native.LogPath)) {
|
||||
logStream.WriteLine($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{logType}] {logLine}");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_ = logSyncSemaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
internal static void InternalWriteLog(
|
||||
string beforeTime,
|
||||
string logLine,
|
||||
LogType logType
|
||||
)
|
||||
{
|
||||
logSyncSemaphore.Wait();
|
||||
if (!File.Exists(Native.LogPath))
|
||||
{
|
||||
FileStream stream = File.Create(Native.LogPath);
|
||||
stream.Close();
|
||||
}
|
||||
|
||||
using (StreamWriter logStream = File.AppendText(Native.LogPath))
|
||||
{
|
||||
logStream.WriteLine(beforeTime);
|
||||
logStream.WriteLine($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff}] [{logType}] {logLine}");
|
||||
}
|
||||
_ = logSyncSemaphore.Release(1);
|
||||
}
|
||||
|
||||
internal static void SetLogLevel(
|
||||
LogType logLevel
|
||||
)
|
||||
{
|
||||
_currentLogLevel = logLevel;
|
||||
}
|
||||
|
||||
private static void RotateLogFile()
|
||||
{
|
||||
if (File.Exists(Native.LogPath) && new FileInfo(Native.LogPath).Length > MaxLogFileSize)
|
||||
{
|
||||
string backupFile = Native.LogPath + ".backup";
|
||||
if (File.Exists(backupFile)) {
|
||||
File.Delete(backupFile);
|
||||
}
|
||||
File.Move(Native.LogPath, backupFile);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="Native"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Internal
|
||||
{
|
||||
internal static unsafe partial class Native
|
||||
{
|
||||
|
||||
internal const string LibName =
|
||||
"StgSharp.Native";
|
||||
|
||||
internal const string LogPath =
|
||||
"SS_error.log";
|
||||
|
||||
internal static IntPtr _libPtr = IntPtr.Zero;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="public"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
public enum LogType
|
||||
{
|
||||
|
||||
Error,
|
||||
Warning,
|
||||
Info,
|
||||
InfoError,
|
||||
InfoWarning,
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="CrossFramePipelineNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.PipeLine;
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public class CrossFrameOperation : IConvertableToPipelineNode
|
||||
{
|
||||
|
||||
private static string[] _inputName = ["TaskIn"];
|
||||
private static string[] _outputName = ["TaskOut"];
|
||||
private Action _mainExecution;
|
||||
private int currentCount;
|
||||
private readonly int cycleCount;
|
||||
|
||||
public CrossFrameOperation(
|
||||
Action startup,
|
||||
int cycleCount
|
||||
)
|
||||
{
|
||||
MainExecution = startup;
|
||||
this.cycleCount = (cycleCount > 1) ?
|
||||
cycleCount :
|
||||
throw new ArgumentException(
|
||||
$"{$"The {typeof(CrossFrameOperation).Name} crosses only one cycle, "}{$"please use a {typeof(PipelineNode).Name} instead."}");
|
||||
this.cycleCount = cycleCount;
|
||||
currentCount = cycleCount;
|
||||
}
|
||||
|
||||
public Action MainExecution
|
||||
{
|
||||
get => _mainExecution;
|
||||
private protected set => _mainExecution = value;
|
||||
}
|
||||
|
||||
public IEnumerable<string> InputPortName => _inputName;
|
||||
|
||||
public IEnumerable<string> OutputPortName => _outputName;
|
||||
|
||||
public PipelineNodeOperation Operation => NodeMain;
|
||||
|
||||
public void NodeMain(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
)
|
||||
{
|
||||
if (currentCount < cycleCount)
|
||||
{
|
||||
Interlocked.Increment(ref currentCount);
|
||||
} else
|
||||
{
|
||||
Interlocked.Exchange(ref currentCount, 0);
|
||||
Task.Run(MainExecution);
|
||||
}
|
||||
|
||||
// Console.WriteLine(currentCount);
|
||||
PipelineNodeOutPort.SkipAll(output);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="IConvertableToPipelineNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public delegate void PipelineNodeOperation(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
);
|
||||
|
||||
public interface IConvertableToPipelineNode
|
||||
{
|
||||
|
||||
public IEnumerable<string> InputPortName { get; }
|
||||
|
||||
public IEnumerable<string> OutputPortName { get; }
|
||||
|
||||
PipelineNodeOperation Operation { get; }
|
||||
|
||||
void NodeMain(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
internal class DefaultConvertableToBlueprintNode : IConvertableToPipelineNode
|
||||
{
|
||||
|
||||
private string[] _input;
|
||||
private string[] _output;
|
||||
private PipelineNodeOperation _execution;
|
||||
|
||||
public DefaultConvertableToBlueprintNode(
|
||||
PipelineNodeOperation execution,
|
||||
string[] input,
|
||||
string[] output
|
||||
)
|
||||
{
|
||||
_input = input;
|
||||
_output = output;
|
||||
_execution = execution;
|
||||
}
|
||||
|
||||
public IEnumerable<string> InputPortName => _input;
|
||||
|
||||
public IEnumerable<string> OutputPortName => _output;
|
||||
|
||||
public PipelineNodeOperation Operation => _execution;
|
||||
|
||||
public void NodeMain(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
)
|
||||
{
|
||||
_execution.Invoke(input, output);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipeLineRunner"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Threading;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public static class PipeLineRunner
|
||||
{
|
||||
|
||||
private static NodeRunner[] allAvailableTask = CreateBlueprintRunnerArray();
|
||||
private static CancellationTokenSource cts = new CancellationTokenSource();
|
||||
private static ConcurrentStack<PipelineScheduler> pipelineInWaiting
|
||||
= new ConcurrentStack<PipelineScheduler>();
|
||||
private static int _callingThread;
|
||||
private static SemaphoreSlim pipelineLock = new SemaphoreSlim(0, 1);
|
||||
|
||||
public static void Run(
|
||||
PipelineScheduler pipeline
|
||||
)
|
||||
{
|
||||
// Console.WriteLine();
|
||||
if (pipeline == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (pipeline.IsRunning) {
|
||||
throw new InvalidOperationException("Attempt to run a blueprint instance twice.");
|
||||
}
|
||||
if (!IsValidCalling()) {
|
||||
pipelineLock.Wait();
|
||||
}
|
||||
if (InterruptAndRunNewPipeline(pipeline))
|
||||
{
|
||||
while (InterruptAndRunNewPipeline(NodeRunner.CurrentPipeline!)) { }
|
||||
}
|
||||
pipelineLock.Release();
|
||||
Interlocked.Exchange(ref _callingThread, -1);
|
||||
}
|
||||
|
||||
public static void SetMaxTaskCount(
|
||||
int size
|
||||
)
|
||||
{
|
||||
if (size < 1) {
|
||||
size = 1;
|
||||
}
|
||||
if (allAvailableTask.Length >= size)
|
||||
{
|
||||
for (int i = size; i < allAvailableTask.Length; i++)
|
||||
{
|
||||
if (allAvailableTask[i] != null) {
|
||||
allAvailableTask[i].Cancel();
|
||||
}
|
||||
}
|
||||
}
|
||||
Array.Resize(ref allAvailableTask, size);
|
||||
}
|
||||
|
||||
public static void Terminate()
|
||||
{
|
||||
cts.Cancel();
|
||||
if (NodeRunner.CurrentPipeline is null) {
|
||||
return;
|
||||
}
|
||||
NodeRunner.CurrentPipeline.Terminate();
|
||||
foreach (NodeRunner task in allAvailableTask) {
|
||||
task.Cts.Cancel();
|
||||
}
|
||||
foreach (PipelineScheduler bp in pipelineInWaiting) {
|
||||
bp.Terminate();
|
||||
}
|
||||
pipelineInWaiting.Clear();
|
||||
}
|
||||
|
||||
internal static bool InterruptAndRunNewPipeline(
|
||||
PipelineScheduler newPipeline
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(newPipeline);
|
||||
if ((newPipeline != NodeRunner.CurrentPipeline) && (NodeRunner.CurrentPipeline != null))
|
||||
{
|
||||
// if a running bp exists
|
||||
pipelineInWaiting.Push(NodeRunner.CurrentPipeline);
|
||||
}
|
||||
|
||||
// load new bp to current bp
|
||||
NodeRunner.CurrentPipeline = newPipeline;
|
||||
Interlocked.Exchange(ref _callingThread, Environment.CurrentManagedThreadId);
|
||||
|
||||
// run beginning node to init it first.
|
||||
if (!NodeRunner.CurrentPipeline.IsRunning)
|
||||
{
|
||||
(NodeRunner.CurrentPipeline.BeginLayer as BeginningNode)!.Run();
|
||||
}
|
||||
|
||||
// refresh all tasks
|
||||
/*
|
||||
for( int i = 0; i < allAvailableTask.Length; i++ )
|
||||
{
|
||||
NodeRunner item = allAvailableTask[ i ];
|
||||
if( ( item is null ) || item.IsCompleted ) {
|
||||
allAvailableTask[ i ] = NodeRunner.Run();
|
||||
}
|
||||
}
|
||||
/**/
|
||||
PipelineNode node;
|
||||
while (!cts.Token.IsCancellationRequested &&
|
||||
NodeRunner.CurrentPipeline.RequestNextMainThreadNode(out node))
|
||||
{
|
||||
node.Run();
|
||||
if (!NodeRunner.CurrentPipeline.IsRunning)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
while ((NodeRunner.CurrentPipeline != null) &&
|
||||
NodeRunner.CurrentPipeline.IsRunning &&
|
||||
!cts.Token.IsCancellationRequested &&
|
||||
NodeRunner.CurrentPipeline!.RequestNexConcurrentNode(out node))
|
||||
{
|
||||
node.Run();
|
||||
if (!NodeRunner.CurrentPipeline!.IsRunning)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
(NodeRunner.CurrentPipeline!.EndLayer as EndingNode)!.Run();
|
||||
bool isEmpty = pipelineInWaiting.TryPop(out PipelineScheduler? pipeline);
|
||||
NodeRunner.CurrentPipeline = pipeline!;
|
||||
return isEmpty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <para> Only the following occasions of calling are valid: </para> <para>1. no bp is
|
||||
/// running or the only bp is quitting.</para><para>2. calling from the main thread or a
|
||||
/// thread the current bp is running on. </para>
|
||||
/// </summary>
|
||||
internal static bool IsValidCalling()
|
||||
{
|
||||
if ((NodeRunner.CurrentPipeline == null) || //no bp running at all
|
||||
((pipelineInWaiting.IsEmpty) && (NodeRunner.CurrentPipeline.IsRunning == false)))
|
||||
{
|
||||
// a bp is quitting
|
||||
// in case no bp is running
|
||||
return true;
|
||||
}
|
||||
int? id = Task.CurrentId;
|
||||
if (id is null)
|
||||
{
|
||||
// in case calling from main thread and current running bp
|
||||
return _callingThread == Environment.CurrentManagedThreadId;
|
||||
}
|
||||
foreach (NodeRunner item in allAvailableTask)
|
||||
{
|
||||
if (item.Id == id)
|
||||
{
|
||||
// if calling from any bp running task
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static NodeRunner[] CreateBlueprintRunnerArray()
|
||||
{
|
||||
int count = Environment.ProcessorCount;
|
||||
return new NodeRunner[ count < 2 ? count : count - 2];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal class NodeRunner : CancellableTask
|
||||
{
|
||||
|
||||
private static volatile PipelineScheduler? _current;
|
||||
|
||||
internal NodeRunner()
|
||||
: base()
|
||||
{
|
||||
_startup = InternalRun;
|
||||
}
|
||||
|
||||
internal static PipelineScheduler? CurrentPipeline
|
||||
{
|
||||
get => _current;
|
||||
set => _current = value;
|
||||
}
|
||||
|
||||
internal static NodeRunner Run()
|
||||
{
|
||||
NodeRunner ret = new NodeRunner
|
||||
{
|
||||
_startup = InternalRun
|
||||
};
|
||||
ret._internalTask = Task.Run(ret.InternalAction);
|
||||
return ret;
|
||||
}
|
||||
|
||||
private static void InternalRun(
|
||||
CancellationToken token
|
||||
)
|
||||
{
|
||||
PipelineNode item;
|
||||
Console.WriteLine($"Global thread {Environment.CurrentManagedThreadId} starts.");
|
||||
while ((CurrentPipeline != null) &&
|
||||
CurrentPipeline.IsRunning &&
|
||||
!token.IsCancellationRequested &&
|
||||
CurrentPipeline.RequestNexConcurrentNode(out item)) {
|
||||
item.Run();
|
||||
}
|
||||
}
|
||||
|
||||
}//------------------------------------- End of Class ------------------------------------------
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipelineMarshal"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public static class PipelineMarshal
|
||||
{
|
||||
|
||||
public static IEnumerable<PipelineNode> GetNodeSortedEnumeration(
|
||||
PipelineScheduler scheduler
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(scheduler);
|
||||
return new PipelineNodeEnumerator(scheduler)
|
||||
{
|
||||
_count = scheduler._allNode.Count,
|
||||
};
|
||||
}
|
||||
|
||||
private class PipelineNodeEnumerator : ICollection<PipelineNode>
|
||||
{
|
||||
|
||||
private readonly List<List<PipelineNode>> _mt, _c;
|
||||
|
||||
public int _count;
|
||||
|
||||
public PipelineNodeEnumerator(
|
||||
PipelineScheduler scheduler
|
||||
)
|
||||
{
|
||||
if (!scheduler.TryGetAllNode(out _mt, out _c)) {
|
||||
throw new InvalidOperationException("Binded scheduler is not sorted.");
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsReadOnly => true;
|
||||
|
||||
public int Count => _count;
|
||||
|
||||
public IEnumerator<PipelineNode> GetEnumerator()
|
||||
{
|
||||
for (int i = 1; i < _mt.Count; i++)
|
||||
{
|
||||
foreach (PipelineNode node in _mt[i]) {
|
||||
yield return node;
|
||||
}
|
||||
foreach (PipelineNode node in _c[i]) {
|
||||
yield return node;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ICollection<PipelineNode>.Add(
|
||||
PipelineNode item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
void ICollection<PipelineNode>.Clear()
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
bool ICollection<PipelineNode>.Contains(
|
||||
PipelineNode item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
void ICollection<PipelineNode>.CopyTo(
|
||||
PipelineNode[] array,
|
||||
int arrayIndex
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
bool ICollection<PipelineNode>.Remove(
|
||||
PipelineNode item
|
||||
)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
IEnumerator IEnumerable.GetEnumerator()
|
||||
{
|
||||
return GetEnumerator();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipelineNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.Serialization;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
/// <risk>
|
||||
/// Reset pipeline has risk, this operation is a direct operation to port in current node, the
|
||||
/// port on the other node cannot get the change.
|
||||
/// </risk>
|
||||
#pragma warning disable CA1036
|
||||
public class PipelineNode
|
||||
#pragma warning restore CA1036
|
||||
{
|
||||
|
||||
private static readonly Dictionary<string, PipelineConnector> _emptyLayer = [];
|
||||
private static PipelineNode _emptyNode = new PipelineNode(
|
||||
PipelineNodeLabel.Empty, null!, false);
|
||||
private IConvertableToPipelineNode _mainOperation;
|
||||
private PipelineNodeLabel _label;
|
||||
private protected bool _nativeRequested;
|
||||
private protected Dictionary<string, PipelineNodeInPort> _input;
|
||||
private protected Dictionary<string, PipelineNodeOutPort> _output;
|
||||
private protected int _level, _count;
|
||||
|
||||
internal PipelineNode(
|
||||
PipelineNodeLabel label,
|
||||
IConvertableToPipelineNode node,
|
||||
bool isNative
|
||||
)
|
||||
{
|
||||
_label = label;
|
||||
_nativeRequested = isNative;
|
||||
_mainOperation = node;
|
||||
InputPorts = new();
|
||||
OutputPorts = new();
|
||||
foreach (string port in node.InputPortName) {
|
||||
InputPorts.Add(port, new PipelineNodeInPort(this, port));
|
||||
}
|
||||
foreach (string port in node.OutputPortName) {
|
||||
OutputPorts.Add(port, new PipelineNodeOutPort(this, port));
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CS8618
|
||||
internal PipelineNode(
|
||||
PipelineNodeOperation operation,
|
||||
PipelineNodeLabel name,
|
||||
bool isNative,
|
||||
string[] inputNames,
|
||||
string[] outputNames
|
||||
)
|
||||
#pragma warning restore CS8618
|
||||
{
|
||||
if ((inputNames == null) || (inputNames.Length == 0)) {
|
||||
throw new ArgumentNullException(nameof(inputNames));
|
||||
}
|
||||
if ((outputNames == null) || (outputNames.Length == 0)) {
|
||||
throw new ArgumentNullException(nameof(outputNames));
|
||||
}
|
||||
_label = name;
|
||||
_nativeRequested = isNative;
|
||||
if (operation == null) {
|
||||
operation = DefaultOperation;
|
||||
}
|
||||
_mainOperation = new DefaultConvertableToBlueprintNode(
|
||||
operation, inputNames, outputNames);
|
||||
InputPorts = new Dictionary<string, PipelineNodeInPort>();
|
||||
OutputPorts = new Dictionary<string, PipelineNodeOutPort>();
|
||||
foreach (string port in inputNames) {
|
||||
InputPorts.Add(port, new PipelineNodeInPort(this, port));
|
||||
}
|
||||
foreach (string port in outputNames) {
|
||||
OutputPorts.Add(port, new PipelineNodeOutPort(this, port));
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsNative
|
||||
{
|
||||
get => _nativeRequested;
|
||||
internal set => _nativeRequested = value;
|
||||
}
|
||||
|
||||
public static Dictionary<string, PipelineConnector> EmptyPortLayer => _emptyLayer;
|
||||
|
||||
public IConvertableToPipelineNode NodeOperation => _mainOperation;
|
||||
|
||||
public int Level
|
||||
{
|
||||
get => _level;
|
||||
private set => _level = value;
|
||||
}
|
||||
|
||||
public static PipelineNode Empty => _emptyNode;
|
||||
|
||||
public PipelineNodeLabel Label => _label;
|
||||
|
||||
internal Dictionary<string, PipelineNodeInPort> InputPorts
|
||||
{
|
||||
get => _input;
|
||||
private set => _input = value;
|
||||
}
|
||||
|
||||
internal Dictionary<string, PipelineNodeOutPort> OutputPorts
|
||||
{
|
||||
get => _output;
|
||||
private set => _output = value;
|
||||
}
|
||||
|
||||
internal HashSet<PipelineNode> Previous { get; } = [];
|
||||
|
||||
internal HashSet<PipelineNode> Next { get; } = [];
|
||||
|
||||
internal IConvertableToPipelineNode Value => _mainOperation;
|
||||
|
||||
internal int RemainingIncreaseCount => Previous.Count - _count;
|
||||
|
||||
public static void Connect(
|
||||
PipelineNode front,
|
||||
string frontNodePort,
|
||||
string backNodePort,
|
||||
PipelineNode back
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(front);
|
||||
ArgumentNullException.ThrowIfNull(back);
|
||||
if (!front.OutputPorts
|
||||
.TryGetValue(frontNodePort,
|
||||
out PipelineNodeOutPort? frontPort)) {
|
||||
throw new ArgumentOutOfRangeException($"Cannot find port named {frontNodePort}");
|
||||
}
|
||||
if (!back.InputPorts.TryGetValue(backNodePort, out PipelineNodeInPort? backPort)) {
|
||||
throw new ArgumentOutOfRangeException($"Cannot find port named {backNodePort}");
|
||||
}
|
||||
backPort.Connect(frontPort);
|
||||
frontPort.Connect(backPort);
|
||||
}
|
||||
|
||||
public IEnumerator<PipelineNode> GetFormerNodes()
|
||||
{
|
||||
foreach (PipelineNodeInPort port in InputPorts.Values)
|
||||
{
|
||||
IEnumerator<PipelineNode> enumerator = port.GetFormerNodes();
|
||||
while (enumerator.MoveNext()) {
|
||||
yield return enumerator.Current;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public virtual void Run()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Console.WriteLine(_label);
|
||||
PipelineNodeInPort.WaitAll(InputPorts);
|
||||
if (_mainOperation != null) {
|
||||
_mainOperation.NodeMain(in _input, in _output);
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
internal bool TryIncreaseLevelTo(
|
||||
int level
|
||||
)
|
||||
{
|
||||
_count++;
|
||||
if (_count > Previous.Count) {
|
||||
return false;
|
||||
}
|
||||
Level = int.Max(level, this.Level);
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static void DefaultOperation(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
)
|
||||
{
|
||||
PipelineNodeOutPort.SkipAll(output);
|
||||
}
|
||||
|
||||
#region port operation
|
||||
|
||||
public PipelineNodeInPort GetInputPort(
|
||||
string lable
|
||||
)
|
||||
{
|
||||
if (!InputPorts.TryGetValue(lable, out PipelineNodeInPort? ret)) {
|
||||
throw new ArgumentOutOfRangeException(
|
||||
$"Cannot find the interface named {lable} in node {Label}.");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public PipelineNodeOutPort GetOutputPort(
|
||||
string lable
|
||||
)
|
||||
{
|
||||
if (!OutputPorts.TryGetValue(lable, out PipelineNodeOutPort? ret)) {
|
||||
throw new ArgumentOutOfRangeException(
|
||||
$"Cannot find the interface named {lable} in node {Label}.");
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public PipelineNodeInPort? DefineCertainInputPort(
|
||||
string label
|
||||
)
|
||||
{
|
||||
if (InputPorts.ContainsKey(label)) {
|
||||
return null;
|
||||
}
|
||||
PipelineNodeInPort port = new(this, label);
|
||||
InputPorts[label] = port;
|
||||
return port;
|
||||
}
|
||||
|
||||
public PipelineNodeOutPort? DefineCertainOutputPort(
|
||||
string label
|
||||
)
|
||||
{
|
||||
if (OutputPorts.ContainsKey(label)) {
|
||||
return null;
|
||||
}
|
||||
PipelineNodeOutPort port = new(this, label);
|
||||
OutputPorts[label] = port;
|
||||
return port;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}//--------------------------------- end of class -----------------------------------
|
||||
|
||||
public class PipelineNodeRunException : Exception
|
||||
{
|
||||
|
||||
public PipelineNodeRunException() { }
|
||||
|
||||
public PipelineNodeRunException(
|
||||
Exception ex
|
||||
)
|
||||
: base(string.Empty, ex) { }
|
||||
|
||||
public PipelineNodeRunException(
|
||||
string message
|
||||
)
|
||||
: base(message) { }
|
||||
|
||||
public PipelineNodeRunException(
|
||||
string message,
|
||||
Exception innerException
|
||||
)
|
||||
: base(message, innerException) { }
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipelineNodeLabel"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public abstract record class PipelineNodeLabel
|
||||
{
|
||||
|
||||
public static PipelineNodeLabel Empty => PipelineNodeLabelEmpty.Only;
|
||||
|
||||
public abstract int ComputeHashCode();
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return ComputeHashCode();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public record class PipelineNodeStringLabel : PipelineNodeLabel
|
||||
{
|
||||
|
||||
public PipelineNodeStringLabel(
|
||||
string name
|
||||
)
|
||||
{
|
||||
Name = name;
|
||||
}
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
public override int ComputeHashCode()
|
||||
{
|
||||
return string.GetHashCode(Name);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal record class PipelineNodeLabelEmpty : PipelineNodeLabel
|
||||
{
|
||||
|
||||
public static PipelineNodeLabelEmpty Only { get; } = new PipelineNodeLabelEmpty();
|
||||
|
||||
public override int ComputeHashCode()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipelineNodePort"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public class PipelineNodeInPort
|
||||
{
|
||||
|
||||
private HashSet<PipelineNodeOutPort> _linkerSet = new HashSet<PipelineNodeOutPort>();
|
||||
|
||||
public PipelineNodeInPort(
|
||||
PipelineNode container,
|
||||
string label
|
||||
)
|
||||
{
|
||||
Container = container;
|
||||
Label = label;
|
||||
}
|
||||
|
||||
public IEnumerable<PipelineNodeOutPort> LinkedPorts => _linkerSet;
|
||||
|
||||
public PipelineNode Container { get; init; }
|
||||
|
||||
public SemaphoreSlim Waiter { get; init; }
|
||||
|
||||
public string Label { get; init; }
|
||||
|
||||
internal HashSet<PipelineNodeOutPort> Linkers => _linkerSet;
|
||||
|
||||
public void Connect(
|
||||
PipelineNodeOutPort port
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(port);
|
||||
if (_linkerSet.Add(port))
|
||||
{
|
||||
port.Linkers.Add(this);
|
||||
Container.Previous.Add(port.Container);
|
||||
port.Container.Next.Add(Container);
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerator<PipelineNode> GetFormerNodes()
|
||||
{
|
||||
foreach (PipelineNodeOutPort port in _linkerSet) {
|
||||
yield return port.Container;
|
||||
}
|
||||
}
|
||||
|
||||
public void Wait()
|
||||
{
|
||||
PipelineNodeOutPort.WaitAll(_linkerSet);
|
||||
}
|
||||
|
||||
public static void WaitAll(
|
||||
IDictionary<string, PipelineNodeInPort> ports
|
||||
)
|
||||
{
|
||||
if (ports is null) {
|
||||
return;
|
||||
}
|
||||
foreach (PipelineNodeInPort port in ports.Values) {
|
||||
port.Wait();
|
||||
}
|
||||
}
|
||||
|
||||
public static void WaitAll(
|
||||
IEnumerable<PipelineNodeInPort> ports
|
||||
)
|
||||
{
|
||||
if (ports is null) {
|
||||
return;
|
||||
}
|
||||
foreach (PipelineNodeInPort port in ports) {
|
||||
port.Wait();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public class PipelineNodeOutPort
|
||||
{
|
||||
|
||||
private HashSet<PipelineNodeInPort> _linkerSet = new HashSet<PipelineNodeInPort>();
|
||||
private volatile int _passCount;
|
||||
private SemaphoreSlim formerCompleteSemaphore;
|
||||
|
||||
public PipelineNodeOutPort(
|
||||
PipelineNode container,
|
||||
string label
|
||||
)
|
||||
{
|
||||
Container = container;
|
||||
Label = label;
|
||||
formerCompleteSemaphore = new SemaphoreSlim(initialCount:0, maxCount:1);
|
||||
}
|
||||
|
||||
public PipelineNode Container { get; init; }
|
||||
|
||||
public string Label { get; init; }
|
||||
|
||||
internal HashSet<PipelineNodeInPort> Linkers => _linkerSet;
|
||||
|
||||
private SemaphoreSlim Waiter => formerCompleteSemaphore;
|
||||
|
||||
public void Connect(
|
||||
PipelineNodeInPort port
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(port);
|
||||
if (_linkerSet.Add(port))
|
||||
{
|
||||
port.Linkers.Add(this);
|
||||
Container.Next.Add(port.Container);
|
||||
port.Container.Previous.Add(Container);
|
||||
}
|
||||
}
|
||||
|
||||
public void Release()
|
||||
{
|
||||
if (_passCount == 0)
|
||||
{
|
||||
Waiter.Release();
|
||||
} else
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public static void SkipAll(
|
||||
Dictionary<string, PipelineNodeOutPort> ports
|
||||
)
|
||||
{
|
||||
if (ports is null) {
|
||||
return;
|
||||
}
|
||||
foreach (PipelineNodeOutPort port in ports.Values) {
|
||||
port.Release();
|
||||
}
|
||||
}
|
||||
|
||||
public void TransmitValue(
|
||||
IPipeLineConnectionPayload value
|
||||
)
|
||||
{
|
||||
foreach (PipelineNodeInPort item in _linkerSet) { }
|
||||
}
|
||||
|
||||
public static void WaitAll(
|
||||
IEnumerable<PipelineNodeOutPort> ports
|
||||
)
|
||||
{
|
||||
if (ports is null) {
|
||||
return;
|
||||
}
|
||||
foreach (PipelineNodeOutPort port in ports) {
|
||||
port.Wait();
|
||||
}
|
||||
}
|
||||
|
||||
internal void Wait()
|
||||
{
|
||||
Waiter.Wait();
|
||||
if (_passCount >= _linkerSet.Count)
|
||||
{
|
||||
Interlocked.Exchange(ref _passCount, 0);
|
||||
} else
|
||||
{
|
||||
Interlocked.Increment(ref _passCount);
|
||||
Waiter.Release();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PipelineScheduler"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Xml;
|
||||
using System.Xml.Linq;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
/// <summary>
|
||||
/// A sets of complex logic designed for running in multi threads environment. Warning: do not
|
||||
/// run a <see cref="PipelineScheduler" /> instance in multi threads more than once.
|
||||
/// </summary>
|
||||
public partial class PipelineScheduler : IConvertableToPipelineNode
|
||||
{
|
||||
|
||||
private BeginningNode _beginning;
|
||||
private bool _isAvailable = false;
|
||||
private EndingNode _ending;
|
||||
|
||||
private int mtLevel, cLevel,mtIndex, cIndex;
|
||||
private List<List<PipelineNode>> _concurrentNodes, _mainThreadNodes ;
|
||||
private SemaphoreSlim runningStat,_concurrentLock = new SemaphoreSlim(1, 1);
|
||||
internal Dictionary<PipelineNodeLabel, PipelineNode> _allNode;
|
||||
|
||||
public PipelineScheduler()
|
||||
{
|
||||
_allNode = new Dictionary<PipelineNodeLabel, PipelineNode>();
|
||||
runningStat = new SemaphoreSlim(1, 1);
|
||||
_beginning = new BeginningNode(this);
|
||||
_ending = new EndingNode(this);
|
||||
}
|
||||
/**/
|
||||
|
||||
public PipelineNode this[
|
||||
PipelineNodeLabel label
|
||||
] => _allNode[label];
|
||||
|
||||
public bool IsRunning => runningStat.CurrentCount == 0;
|
||||
|
||||
public IEnumerable<string> InputPortName => _beginning.InputPorts.Keys;
|
||||
|
||||
public IEnumerable<string> OutputPortName => _beginning.OutputPorts.Keys;
|
||||
|
||||
public PipelineNode BeginLayer => _beginning;
|
||||
|
||||
public PipelineNode EndLayer => _ending;
|
||||
|
||||
public PipelineNodeOperation Operation => (this as IConvertableToPipelineNode).NodeMain;
|
||||
|
||||
/**/
|
||||
|
||||
internal SemaphoreSlim RunningStat => runningStat;
|
||||
|
||||
public void AddNode(
|
||||
PipelineNode node,
|
||||
bool isNative
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(node);
|
||||
node.IsNative = isNative;
|
||||
_allNode.TryAdd(node.Label, node);
|
||||
}
|
||||
|
||||
public void ArrangeNodes()
|
||||
{
|
||||
if (_isAvailable) {
|
||||
return;
|
||||
}
|
||||
int maxLevel = 1;
|
||||
Queue<PipelineNode> completedNodes = new Queue<PipelineNode>();
|
||||
foreach (PipelineNode item in _beginning.Next)
|
||||
{
|
||||
item.TryIncreaseLevelTo(1);
|
||||
completedNodes.Enqueue(item);
|
||||
}
|
||||
|
||||
// calc level of nodes O(V)
|
||||
while (completedNodes.Count > 0)
|
||||
{
|
||||
PipelineNode u = completedNodes.Dequeue();
|
||||
foreach (PipelineNode v in u.Next)
|
||||
{
|
||||
if (!v.TryIncreaseLevelTo(u.Level + 1)) {
|
||||
throw new InvalidOperationException("Ring in DAG detected.");
|
||||
}
|
||||
if (v.RemainingIncreaseCount == 0)
|
||||
{
|
||||
completedNodes.Enqueue(v);
|
||||
maxLevel = int.Max(maxLevel, v.Level);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bucket sort of nodes, O(E)
|
||||
_concurrentNodes = new List<List<PipelineNode>>(maxLevel + 1);
|
||||
_mainThreadNodes = new List<List<PipelineNode>>(maxLevel + 1);
|
||||
for (int i = 0; i <= maxLevel; i++)
|
||||
{
|
||||
_concurrentNodes.Add([]);
|
||||
_mainThreadNodes.Add([]);
|
||||
}
|
||||
foreach (PipelineNode item in _allNode.Values)
|
||||
{
|
||||
if (item.IsNative)
|
||||
{
|
||||
_mainThreadNodes[item.Level].Add(item);
|
||||
} else
|
||||
{
|
||||
_concurrentNodes[item.Level].Add(item);
|
||||
}
|
||||
}
|
||||
foreach (List<PipelineNode> item in _concurrentNodes) {
|
||||
item.TrimExcess();
|
||||
}
|
||||
foreach (List<PipelineNode> item in _mainThreadNodes) {
|
||||
item.TrimExcess();
|
||||
}
|
||||
_isAvailable = true;
|
||||
}
|
||||
|
||||
public PipelineNode Create(
|
||||
PipelineNodeLabel label,
|
||||
[NotNull] IConvertableToPipelineNode body
|
||||
)
|
||||
{
|
||||
if (_allNode.TryGetValue(label, out PipelineNode? node)) {
|
||||
throw new InvalidOperationException("Label has exist in scheduler.");
|
||||
}
|
||||
node = new PipelineNode(label, body, false);
|
||||
_allNode.Add(label, node);
|
||||
return node;
|
||||
}
|
||||
|
||||
public PipelineNode GetNode(
|
||||
PipelineNodeLabel label
|
||||
)
|
||||
{
|
||||
if (_allNode.TryGetValue(label, out PipelineNode? node))
|
||||
{
|
||||
return node;
|
||||
} else
|
||||
{
|
||||
throw new KeyNotFoundException();
|
||||
}
|
||||
}
|
||||
|
||||
public void LinkToInput(
|
||||
PipelineNodeInPort port
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(port);
|
||||
PipelineNodeOutPort p = _beginning.DefaultOut;
|
||||
p.Connect(port);
|
||||
}
|
||||
|
||||
public void LinkToInput(
|
||||
PipelineNodeInPort port,
|
||||
string inputPortName
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(port);
|
||||
PipelineNodeOutPort p = _beginning.DefineCertainOutputPort(inputPortName);
|
||||
p.Connect(port);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void SetInput(
|
||||
params (string label, IPipeLineConnectionPayload arg)[] parameters
|
||||
)
|
||||
{
|
||||
_beginning.SetInputData(parameters);
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
_concurrentLock.Wait();
|
||||
(mtLevel, cLevel, mtIndex, cIndex) = (int.MaxValue, int.MaxValue, int.MaxValue, int.MaxValue);
|
||||
_concurrentLock.Release();
|
||||
}
|
||||
|
||||
public bool TryGetAllNode(
|
||||
out List<List<PipelineNode>> mainThreadNodes,
|
||||
out List<List<PipelineNode>> concurrentNodes
|
||||
)
|
||||
{
|
||||
if (_isAvailable)
|
||||
{
|
||||
mainThreadNodes = _mainThreadNodes;
|
||||
concurrentNodes = _concurrentNodes;
|
||||
return true;
|
||||
}
|
||||
mainThreadNodes = null;
|
||||
concurrentNodes = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
internal bool RequestNexConcurrentNode(
|
||||
out PipelineNode node
|
||||
)
|
||||
{
|
||||
int index, level;
|
||||
if (cLevel >= _concurrentNodes.Count)
|
||||
{
|
||||
node = null!;
|
||||
return false;
|
||||
}
|
||||
_concurrentLock.Wait();
|
||||
while (cIndex >= _concurrentNodes[cLevel].Count)
|
||||
{
|
||||
Interlocked.Increment(ref cLevel);
|
||||
Interlocked.Exchange(ref cIndex, 0);
|
||||
if (cLevel >= _concurrentNodes.Count)
|
||||
{
|
||||
node = null!;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
index = cIndex;
|
||||
level = cLevel;
|
||||
Interlocked.Increment(ref cIndex);
|
||||
_concurrentLock.Release();
|
||||
node = _concurrentNodes[level][index];
|
||||
return true;
|
||||
}
|
||||
|
||||
internal bool RequestNextMainThreadNode(
|
||||
out PipelineNode node
|
||||
)
|
||||
{
|
||||
if (mtLevel >= _mainThreadNodes.Count)
|
||||
{
|
||||
node = null!;
|
||||
return false;
|
||||
}
|
||||
while (mtIndex >= _mainThreadNodes[cLevel].Count)
|
||||
{
|
||||
Interlocked.Increment(ref mtLevel);
|
||||
Interlocked.Exchange(ref mtIndex, 0);
|
||||
if (mtLevel >= _mainThreadNodes.Count)
|
||||
{
|
||||
node = null!;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
node = _mainThreadNodes[mtLevel][mtIndex];
|
||||
return true;
|
||||
}
|
||||
|
||||
internal void ResetIndex()
|
||||
{
|
||||
(mtLevel, cLevel, mtIndex, cIndex) = (0, 0, 0, 0);
|
||||
_concurrentLock.Release();
|
||||
}
|
||||
|
||||
void IConvertableToPipelineNode.NodeMain(
|
||||
in Dictionary<string, PipelineNodeInPort> input,
|
||||
in Dictionary<string, PipelineNodeOutPort> output
|
||||
)
|
||||
{
|
||||
PipeLineRunner.Run(this);
|
||||
PipelineNodeOutPort.SkipAll(output);
|
||||
}
|
||||
|
||||
~PipelineScheduler()
|
||||
{
|
||||
runningStat.Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="PortArgs"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Security.Authentication.ExtendedProtection;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
public class PipelineConnector
|
||||
{
|
||||
|
||||
private protected IPipeLineConnectionPayload args;
|
||||
private protected PipelineNodeInPort output;
|
||||
private protected PipelineNodeOutPort input;
|
||||
private protected SemaphoreSlim formerCompleteSemaphore;
|
||||
|
||||
public PipelineConnector(
|
||||
PipelineNodeOutPort former,
|
||||
PipelineNodeInPort after
|
||||
)
|
||||
{
|
||||
this.input = former;
|
||||
this.output = after;
|
||||
formerCompleteSemaphore = new SemaphoreSlim(initialCount:0, maxCount:1);
|
||||
}
|
||||
|
||||
public IPipeLineConnectionPayload Args
|
||||
{
|
||||
get => args;
|
||||
set => args = value;
|
||||
}
|
||||
|
||||
public PipelineNodeInPort Output => output;
|
||||
|
||||
public PipelineNodeOutPort Input => input;
|
||||
|
||||
public static void SkipAll(
|
||||
Dictionary<string, PipelineConnector> ports
|
||||
)
|
||||
{
|
||||
if (ports == null) {
|
||||
return;
|
||||
}
|
||||
foreach (KeyValuePair<string, PipelineConnector> item in ports)
|
||||
{
|
||||
if (item.Value != null) {
|
||||
item.Value.CompleteAndSkip();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void WaitAll(
|
||||
Dictionary<string, PipelineConnector> ports
|
||||
)
|
||||
{
|
||||
if (ports == null) {
|
||||
return;
|
||||
}
|
||||
foreach (KeyValuePair<string, PipelineConnector> item in ports)
|
||||
{
|
||||
if (item.Value != null) {
|
||||
item.Value.WaitForStart();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public virtual void WaitForStart()
|
||||
{
|
||||
// Console.WriteLine($"Waiting \t{input.ContextName}\t->\t{output.ContextName}");
|
||||
formerCompleteSemaphore.Wait();
|
||||
}
|
||||
|
||||
~PipelineConnector()
|
||||
{
|
||||
formerCompleteSemaphore.Dispose();
|
||||
}
|
||||
|
||||
#region pipline op
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void CompleteAndSkip()
|
||||
{
|
||||
// Console.WriteLine($"Complete\t{input.Name}\t->\t{output.Name}");
|
||||
formerCompleteSemaphore.Release();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void CompleteAndWriteData(
|
||||
IPipeLineConnectionPayload args
|
||||
)
|
||||
{
|
||||
this.args = args;
|
||||
CompleteAndSkip();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
|
||||
public abstract class IPipeLineConnectionPayload { }
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SideNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Security.Principal;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.PipeLine
|
||||
{
|
||||
internal sealed class BeginningNode : PipelineNode
|
||||
{
|
||||
|
||||
public const string OutName = "default";
|
||||
|
||||
private PipelineScheduler _bp;
|
||||
|
||||
public BeginningNode(
|
||||
PipelineScheduler bp
|
||||
)
|
||||
: base(DefaultOperation, new PipelineNodeStringLabel("BeginningNode"), true,
|
||||
["default"], [OutName])
|
||||
{
|
||||
_bp = bp;
|
||||
}
|
||||
|
||||
public PipelineNodeOutPort DefaultOut => OutputPorts[OutName];
|
||||
|
||||
public override void Run()
|
||||
{
|
||||
_bp.RunningStat.Wait();
|
||||
PipelineNodeOutPort.SkipAll(_output);
|
||||
}
|
||||
|
||||
internal void SetInputData(
|
||||
IEnumerable<(string, IPipeLineConnectionPayload)> data
|
||||
)
|
||||
{
|
||||
if (data is null) {
|
||||
return;
|
||||
}
|
||||
foreach ((string, IPipeLineConnectionPayload) item in data) {
|
||||
OutputPorts[item.Item1].TransmitValue(item.Item2);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal sealed class EndingNode : PipelineNode
|
||||
{
|
||||
|
||||
private PipelineScheduler _bp;
|
||||
|
||||
public EndingNode(
|
||||
PipelineScheduler bp
|
||||
)
|
||||
: base(DefaultOperation, name:new PipelineNodeStringLabel("EndingNode"), true,
|
||||
["default"], ["default"])
|
||||
{
|
||||
_bp = bp;
|
||||
}
|
||||
|
||||
public override void Run()
|
||||
{
|
||||
PipelineNodeInPort.WaitAll(_input);
|
||||
_bp.ResetIndex();
|
||||
_bp.RunningStat.Release();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- https://go.microsoft.com/fwlink/?LinkID=208121. -->
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
<PublishDir>C:\Users\Nitlo\OneDrive - MSFT\桌面\新建文件夹</PublishDir>
|
||||
<PublishProtocol>FileSystem</PublishProtocol>
|
||||
<_TargetId>Folder</_TargetId>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
|
||||
<SelfContained>true</SelfContained>
|
||||
<PublishSingleFile>true</PublishSingleFile>
|
||||
<PublishReadyToRun>true</PublishReadyToRun>
|
||||
<PublishTrimmed>true</PublishTrimmed>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,36 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ISimpleStateMachine"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.State
|
||||
{
|
||||
public interface ISimpleStateMachine<TStateNode, TStateLabel, TStateContext>
|
||||
where TStateNode : SimpleStateMachineNode<TStateNode, TStateLabel, TStateContext>
|
||||
where TStateLabel : unmanaged, IEquatable<TStateLabel>
|
||||
where TStateContext : StateContext
|
||||
{
|
||||
|
||||
TStateNode this[
|
||||
TStateLabel label
|
||||
] { get; }
|
||||
|
||||
ref readonly TStateNode GetCurrentStateNode();
|
||||
|
||||
void Reset();
|
||||
|
||||
bool TryMoveNext(
|
||||
TStateContext ctx
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SimpleStateMachineNode"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Numerics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.State
|
||||
{
|
||||
public abstract class SimpleStateMachineNode<TSelf, TLabel, TStateContext>
|
||||
where TSelf : SimpleStateMachineNode<TSelf, TLabel, TStateContext>
|
||||
where TLabel : unmanaged, IEquatable<TLabel>
|
||||
where TStateContext : StateContext
|
||||
{
|
||||
|
||||
private readonly List<int> _jumpTable = [];
|
||||
|
||||
public ReadOnlySpan<int> AvailableNodes => CollectionsMarshal.AsSpan(_jumpTable);
|
||||
|
||||
public abstract bool IsValidateFrom(
|
||||
TSelf former,
|
||||
TStateContext context
|
||||
);
|
||||
|
||||
public abstract bool TryTransaction(
|
||||
ReadOnlySpan<TSelf> nodeSpan,
|
||||
TStateContext context,
|
||||
out TLabel label
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="StateContext"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.State
|
||||
{
|
||||
public abstract class StateContext { }
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Library</OutputType>
|
||||
<IsNativeHost>true</IsNativeHost>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Remove="Graphics\**" />
|
||||
<EmbeddedResource Remove="Graphics\**" />
|
||||
<None Remove="Graphics\**" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,347 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="AdaptiveBackoffStrategy"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
/// <summary>
|
||||
/// Efficient adaptive backoff strategy that dynamically selects optimal backoff methods based on
|
||||
/// system load and success rate
|
||||
/// </summary>
|
||||
public struct AdaptiveBackoffStrategy
|
||||
{
|
||||
|
||||
private const int EVALUATION_INTERVAL_MS = 2000; // Global re-evaluation interval
|
||||
private const int EXPLORATION_PROBABILITY = 10; // 10% probability to explore non-optimal strategies
|
||||
private const int FAST_CONVERGENCE_THRESHOLD = 1000; // Fast convergence threshold
|
||||
|
||||
// Configuration constants
|
||||
private const int INITIAL_SPINWAIT_THRESHOLD = 8; // Cheap spin count for first few SpinWait iterations
|
||||
private const int MIN_SAMPLES_FOR_EVALUATION = 100; // Minimum sample count before starting evaluation
|
||||
|
||||
// Global shared state (all threads share learning results)
|
||||
private static readonly PerformanceStats[] s_globalStats = new PerformanceStats[3];
|
||||
private static volatile BackoffMethod s_globalBestMethod;
|
||||
private static volatile int s_evaluationInProgress;
|
||||
private static long s_lastGlobalEvaluationTicks;
|
||||
private BackoffMethod _currentMethod;
|
||||
private int _consecutiveFailures;
|
||||
private int _localAttempts;
|
||||
|
||||
// Instance state
|
||||
private SpinWait _spinWait;
|
||||
private uint _fastRng; // Lightweight pseudo-random number generator
|
||||
|
||||
// Static constructor: initialize optimal strategy based on system characteristics
|
||||
static AdaptiveBackoffStrategy()
|
||||
{
|
||||
// Based on your benchmark results:
|
||||
// - Low core count (<= 8): SpinWait has clear advantage
|
||||
// - Medium core count (<= 16): Close performance, slightly favor SpinWait
|
||||
// - High core count (>= 20): Sleep0 has significant advantage under high concurrency
|
||||
int logicalCores = Environment.ProcessorCount;
|
||||
|
||||
if (logicalCores <= 8)
|
||||
{
|
||||
s_globalBestMethod = BackoffMethod.SpinWait;
|
||||
} else if (logicalCores <= 16)
|
||||
{
|
||||
s_globalBestMethod = BackoffMethod.SpinWait; // Default conservative choice
|
||||
} else
|
||||
{
|
||||
// High core count systems, based on your 64-thread benchmark results
|
||||
s_globalBestMethod = BackoffMethod.Sleep0;
|
||||
}
|
||||
|
||||
s_lastGlobalEvaluationTicks = Environment.TickCount64;
|
||||
|
||||
// Initialize statistics array
|
||||
for (int i = 0; i < s_globalStats.Length; i++)
|
||||
{
|
||||
s_globalStats[i].Reset();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the currently used backoff method (for debugging/monitoring)
|
||||
/// </summary>
|
||||
public readonly BackoffMethod CurrentMethod => _currentMethod;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the globally considered optimal backoff method
|
||||
/// </summary>
|
||||
public static BackoffMethod GlobalBestMethod => s_globalBestMethod;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new adaptive backoff strategy instance
|
||||
/// </summary>
|
||||
public static AdaptiveBackoffStrategy Create()
|
||||
{
|
||||
return new AdaptiveBackoffStrategy
|
||||
{
|
||||
_spinWait = default,
|
||||
_currentMethod = s_globalBestMethod,
|
||||
_consecutiveFailures = 0,
|
||||
_localAttempts = 0,
|
||||
_fastRng =
|
||||
(uint)(Environment.TickCount64 ^ Thread.CurrentThread.ManagedThreadId * 0x9E3779B9U)
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets global statistics for the specified strategy (for monitoring)
|
||||
/// </summary>
|
||||
public static (long attempts, long successes, double successRate) GetGlobalStats(
|
||||
BackoffMethod method
|
||||
)
|
||||
{
|
||||
PerformanceStats stats = s_globalStats[(int)method];
|
||||
return (stats.Attempts, stats.Successes, stats.SuccessRate);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Called after CAS failure, performs backoff and may switch strategies
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void OnFailureAndBackoff()
|
||||
{
|
||||
_consecutiveFailures++;
|
||||
|
||||
// Record failure
|
||||
s_globalStats[(int)_currentMethod].RecordAttempt(success:false);
|
||||
|
||||
// Periodically re-evaluate global optimal strategy
|
||||
if (ShouldEvaluateGlobalStrategy())
|
||||
{
|
||||
EvaluateAndUpdateGlobalStrategy();
|
||||
}
|
||||
|
||||
// Decide whether to switch strategies based on failure count and current strategy
|
||||
if (_consecutiveFailures >= INITIAL_SPINWAIT_THRESHOLD && ShouldSwitchStrategy())
|
||||
{
|
||||
SwitchToOptimalStrategy();
|
||||
}
|
||||
|
||||
// Perform the actual backoff operation
|
||||
PerformBackoff();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Called after CAS success, resets state
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void OnSuccess()
|
||||
{
|
||||
// Record success
|
||||
s_globalStats[(int)_currentMethod].RecordAttempt(success:true);
|
||||
|
||||
// Reset state
|
||||
_spinWait = default;
|
||||
_consecutiveFailures = 0;
|
||||
|
||||
// Fast convergence: tend to keep current strategy when continuously successful
|
||||
if (_localAttempts < FAST_CONVERGENCE_THRESHOLD)
|
||||
{
|
||||
_localAttempts++;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates and updates the global optimal strategy
|
||||
/// </summary>
|
||||
private static void EvaluateAndUpdateGlobalStrategy()
|
||||
{
|
||||
// Use CAS to ensure only one thread performs evaluation
|
||||
if (Interlocked.CompareExchange(ref s_evaluationInProgress, 1, 0) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
long now = Environment.TickCount64;
|
||||
|
||||
// Double-check to avoid duplicate evaluation
|
||||
if (now - Volatile.Read(ref s_lastGlobalEvaluationTicks) <= EVALUATION_INTERVAL_MS)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
BackoffMethod bestMethod = s_globalBestMethod;
|
||||
double bestSuccessRate = -1.0;
|
||||
|
||||
// Find the strategy with the highest success rate
|
||||
for (int i = 0; i < s_globalStats.Length; i++)
|
||||
{
|
||||
PerformanceStats stats = s_globalStats[i];
|
||||
if (stats.Attempts >= MIN_SAMPLES_FOR_EVALUATION)
|
||||
{
|
||||
double successRate = stats.SuccessRate;
|
||||
if (successRate > bestSuccessRate)
|
||||
{
|
||||
bestSuccessRate = successRate;
|
||||
bestMethod = (BackoffMethod)i;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update global optimal strategy
|
||||
s_globalBestMethod = bestMethod;
|
||||
Volatile.Write(ref s_lastGlobalEvaluationTicks, now);
|
||||
|
||||
// Reset statistics (maintain exploratory behavior)
|
||||
for (int i = 0; i < s_globalStats.Length; i++)
|
||||
{
|
||||
s_globalStats[i].Reset();
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Volatile.Write(ref s_evaluationInProgress, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fast pseudo-random number generator (LCG)
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private uint NextRandom()
|
||||
{
|
||||
_fastRng = _fastRng * 1103515245U + 12345U;
|
||||
return _fastRng;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs the actual backoff operation
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce hot path code size
|
||||
private void PerformBackoff()
|
||||
{
|
||||
switch (_currentMethod)
|
||||
{
|
||||
case BackoffMethod.SpinWait:
|
||||
if (_consecutiveFailures <= INITIAL_SPINWAIT_THRESHOLD)
|
||||
{
|
||||
_spinWait.SpinOnce(sleep1Threshold:-1);
|
||||
} else if (_spinWait.NextSpinWillYield)
|
||||
{
|
||||
int yields = (int)(NextRandom() & 7) + 1; // 1-8 times
|
||||
for (int i = 0; i < yields; i++) {
|
||||
_ = Thread.Yield();
|
||||
}
|
||||
} else
|
||||
{
|
||||
_spinWait.SpinOnce(sleep1Threshold:-1);
|
||||
}
|
||||
break;
|
||||
|
||||
case BackoffMethod.Yield:
|
||||
_ = Thread.Yield();
|
||||
break;
|
||||
|
||||
case BackoffMethod.Sleep0:
|
||||
Thread.Sleep(0);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether global strategy evaluation should be performed
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool ShouldEvaluateGlobalStrategy()
|
||||
{
|
||||
long now = Environment.TickCount64;
|
||||
return now - Volatile.Read(ref s_lastGlobalEvaluationTicks) > EVALUATION_INTERVAL_MS;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether strategy should be switched
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool ShouldSwitchStrategy()
|
||||
{
|
||||
// Fast convergence phase: reduce strategy switching
|
||||
if (_localAttempts < FAST_CONVERGENCE_THRESHOLD && _consecutiveFailures < 32)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Small probability exploration: occasionally try other strategies even when current is optimal
|
||||
return NextRandom() % 100 < EXPLORATION_PROBABILITY || _currentMethod != s_globalBestMethod;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Switches to the currently considered optimal strategy
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void SwitchToOptimalStrategy()
|
||||
{
|
||||
BackoffMethod targetMethod;
|
||||
|
||||
// Most of the time choose the globally optimal
|
||||
if (NextRandom() % 100 >= EXPLORATION_PROBABILITY)
|
||||
{
|
||||
targetMethod = s_globalBestMethod;
|
||||
} else
|
||||
{
|
||||
// Small probability random exploration
|
||||
targetMethod = (BackoffMethod)(NextRandom() % 3);
|
||||
}
|
||||
|
||||
if (targetMethod != _currentMethod)
|
||||
{
|
||||
_currentMethod = targetMethod;
|
||||
_spinWait = default; // Reset SpinWait state when switching strategies
|
||||
_localAttempts = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public enum BackoffMethod : byte
|
||||
{
|
||||
|
||||
SpinWait = 0,
|
||||
Yield = 1,
|
||||
Sleep0 = 2
|
||||
|
||||
}
|
||||
|
||||
private struct PerformanceStats
|
||||
{
|
||||
|
||||
public long Attempts;
|
||||
public long LastResetTicks;
|
||||
public long Successes;
|
||||
|
||||
public readonly double SuccessRate => Attempts == 0 ? 0.0 : (double)Successes / Attempts;
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void RecordAttempt(
|
||||
bool success
|
||||
)
|
||||
{
|
||||
_ = Interlocked.Increment(ref Attempts);
|
||||
if (success) {
|
||||
_ = Interlocked.Increment(ref Successes);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Reset()
|
||||
{
|
||||
Attempts = 0;
|
||||
Successes = 0;
|
||||
LastResetTicks = Environment.TickCount64;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="BufferExpansionLock"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Threading
|
||||
{
|
||||
/// <summary>
|
||||
/// Provide a lock for managing concurrent access to a buffer that can be expanded and use a
|
||||
/// common two-pase-copy expanding mechanism.
|
||||
/// </summary>
|
||||
public class BufferExpansionLock : IDisposable
|
||||
{
|
||||
|
||||
private static readonly ThreadLocal<int> Count = new(() => 0);
|
||||
|
||||
private readonly ReaderWriterLockSlim _copyLock;
|
||||
private readonly ReaderWriterLockSlim _expansionLock;
|
||||
|
||||
public BufferExpansionLock()
|
||||
{
|
||||
_expansionLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
|
||||
_copyLock = new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion);
|
||||
}
|
||||
|
||||
public bool IsThreadReadingMetaData => _expansionLock.IsReadLockHeld;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_expansionLock?.Dispose();
|
||||
_copyLock?.Dispose();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void EnterBufferCopy()
|
||||
{
|
||||
_copyLock.EnterWriteLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void EnterBufferRead()
|
||||
{
|
||||
int c = Count.Value;
|
||||
if (c == 0) {
|
||||
_copyLock.EnterReadLock();
|
||||
}
|
||||
Count.Value = c + 1;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void EnterExpansionProcess()
|
||||
{
|
||||
_expansionLock.EnterWriteLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void EnterMetaDataRead()
|
||||
{
|
||||
_expansionLock.EnterReadLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ExitBufferCopy()
|
||||
{
|
||||
_copyLock.ExitWriteLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ExitBufferRead()
|
||||
{
|
||||
int c = Count.Value - 1;
|
||||
Count.Value = c;
|
||||
if (c == 0) {
|
||||
_copyLock.ExitReadLock();
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ExitExpansionProcess()
|
||||
{
|
||||
_expansionLock.ExitWriteLock();
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void ExitMetaDataRead()
|
||||
{
|
||||
_expansionLock.ExitReadLock();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="CancellableTask"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Threading
|
||||
{
|
||||
public class CancellableTask : IDisposable
|
||||
{
|
||||
|
||||
private bool disposedValue;
|
||||
private CancellationTokenSource cts;
|
||||
private protected Action<CancellationToken> _startup;
|
||||
private protected Task _internalTask;
|
||||
|
||||
private protected CancellableTask()
|
||||
{
|
||||
cts = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
private protected CancellableTask(
|
||||
Action<CancellationToken> action
|
||||
)
|
||||
{
|
||||
_internalTask = new Task(InternalAction);
|
||||
cts = new CancellationTokenSource();
|
||||
}
|
||||
|
||||
public bool IsCompleted => _internalTask.IsCompleted;
|
||||
|
||||
public CancellationTokenSource Cts => cts;
|
||||
|
||||
public int Id => _internalTask.Id;
|
||||
|
||||
public void Cancel()
|
||||
{
|
||||
Cts.Cancel();
|
||||
}
|
||||
|
||||
public void CancelAndWait()
|
||||
{
|
||||
Cts.Cancel();
|
||||
_internalTask.Wait();
|
||||
}
|
||||
|
||||
// ~CancellableTask()
|
||||
// {
|
||||
// Dispose(disposing: false);
|
||||
// }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(disposing:true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public static CancellableTask Run(
|
||||
Action<CancellationToken> startup
|
||||
)
|
||||
{
|
||||
CancellableTask ret = new CancellableTask
|
||||
{
|
||||
_startup = startup
|
||||
};
|
||||
ret._internalTask = Task.Run(ret.InternalAction);
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void Wait()
|
||||
{
|
||||
_internalTask.Wait();
|
||||
}
|
||||
|
||||
public static void WaitAll(
|
||||
IEnumerable<CancellableTask> tasks
|
||||
)
|
||||
{
|
||||
IEnumerable<Task> convertedTasks = tasks.Select<CancellableTask, Task>(
|
||||
ct => ct._internalTask);
|
||||
Task.WaitAll(convertedTasks.ToArray());
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal void InternalAction()
|
||||
{
|
||||
_startup(Cts.Token);
|
||||
}
|
||||
|
||||
protected virtual void Dispose(
|
||||
bool disposing
|
||||
)
|
||||
{
|
||||
if (!disposedValue)
|
||||
{
|
||||
if (disposing)
|
||||
{
|
||||
if (IsCompleted)
|
||||
{
|
||||
cts.Dispose();
|
||||
_internalTask.Dispose();
|
||||
} else
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Cannot dispose a cancellable task before completed");
|
||||
}
|
||||
}
|
||||
|
||||
disposedValue = true;
|
||||
}
|
||||
}
|
||||
|
||||
public static implicit operator Task(
|
||||
CancellableTask task
|
||||
)
|
||||
{
|
||||
return task._internalTask;
|
||||
}
|
||||
|
||||
~CancellableTask()
|
||||
{
|
||||
Cts.Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="SingleConsumerBell"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Threading
|
||||
{
|
||||
/// <summary>
|
||||
/// A single-consumer bell/signal primitive.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>Producers can call <see cref="Set" /> multiple times to publish signals. The bell
|
||||
/// keeps a pending count of not-yet-consumed signals. The consumer uses <see
|
||||
/// cref="Wait(object, CancellationToken)" /> or <see cref="TryWait(object)" /> to consume
|
||||
/// signals. </para> <para>When a producer increments the pending count from 0 to 1, the
|
||||
/// internal event is set to wake the consumer. Consuming via <see cref="TryWait(object)" />
|
||||
/// or <see cref="Wait(object, CancellationToken)" /> clears the entire pending count
|
||||
/// (aggregate consumption). If new signals arrive while resetting, the event is set again to
|
||||
/// avoid missed signals. </para> <para>Ownership: only the specified <c> owner </c> can call
|
||||
/// the consumer APIs. </para>
|
||||
/// </remarks>
|
||||
public sealed class SingleConsumerBell : IDisposable
|
||||
{
|
||||
|
||||
private volatile bool _disposed;
|
||||
private int _pending; // not volatile: used with Interlocked/Volatile APIs to avoid CS0420
|
||||
private readonly ManualResetEventSlim _evt;
|
||||
private readonly object _owner;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of <see cref="SingleConsumerBell" />.
|
||||
/// </summary>
|
||||
/// <param name="owner">
|
||||
/// The unique consumer identity; subsequent calls to Wait/TryWait must pass the same
|
||||
/// reference.
|
||||
/// </param>
|
||||
/// <param name="spinCount">
|
||||
/// Spin count passed to the underlying <see cref="ManualResetEventSlim" />.
|
||||
/// </param>
|
||||
/// <exception cref="ArgumentNullException">
|
||||
/// Thrown if <paramref name="owner" /> is null.
|
||||
/// </exception>
|
||||
public SingleConsumerBell(
|
||||
object owner,
|
||||
int spinCount = 100
|
||||
)
|
||||
{
|
||||
_owner = owner ?? throw new ArgumentNullException(nameof(owner));
|
||||
_evt = new ManualResetEventSlim(false, spinCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the bell, sets the event to wake any waiting consumer, and releases
|
||||
/// resources.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
_evt.Set();
|
||||
_evt.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes a signal by incrementing the not-yet-consumed count and returns the current
|
||||
/// pending count.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// If the pending count transitions from 0 to 1, the event is set to wake the consumer.
|
||||
/// </remarks>
|
||||
/// <returns>
|
||||
/// The current pending count including this call.
|
||||
/// </returns>
|
||||
public int Set()
|
||||
{
|
||||
if (_disposed) {
|
||||
return Volatile.Read(ref _pending);
|
||||
}
|
||||
|
||||
// Increment pending; if we transitioned 0 -> 1, set the event to wake the consumer
|
||||
int newCount = Interlocked.Increment(ref _pending);
|
||||
if (newCount == 1) {
|
||||
_evt.Set();
|
||||
}
|
||||
return newCount;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to consume any pending signals without blocking.
|
||||
/// </summary>
|
||||
/// <param name="consumer">
|
||||
/// The consumer object; must be the same reference provided at construction.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// <c> true </c> if a signal was available (pending or event set); otherwise
|
||||
/// <c>false</c>.
|
||||
/// </returns>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown if <paramref name="consumer" /> is not the owner.
|
||||
/// </exception>
|
||||
public bool TryWait(
|
||||
object consumer
|
||||
)
|
||||
{
|
||||
if (!ReferenceEquals(_owner, consumer)) {
|
||||
throw new InvalidOperationException("This bell is owned by another consumer.");
|
||||
}
|
||||
|
||||
if (!_evt.IsSet && Volatile.Read(ref _pending) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Aggregate consumption: clear pending and reset event
|
||||
_ = Interlocked.Exchange(ref _pending, 0);
|
||||
_evt.Reset();
|
||||
|
||||
// If concurrent Set happened during reset, re-set the event to avoid lost signals
|
||||
if (Volatile.Read(ref _pending) != 0)
|
||||
{
|
||||
_evt.Set();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Blocks until a signal is available, then consumes all pending signals.
|
||||
/// </summary>
|
||||
/// <param name="consumer">
|
||||
/// The consumer object; must be the same reference provided at construction.
|
||||
/// </param>
|
||||
/// <param name="cancellationToken">
|
||||
/// A token to cancel the wait.
|
||||
/// </param>
|
||||
/// <exception cref="InvalidOperationException">
|
||||
/// Thrown if <paramref name="consumer" /> is not the owner.
|
||||
/// </exception>
|
||||
public void Wait(
|
||||
object consumer,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
if (!ReferenceEquals(_owner, consumer)) {
|
||||
throw new InvalidOperationException("This bell is owned by another consumer.");
|
||||
}
|
||||
|
||||
_evt.Wait(cancellationToken);
|
||||
|
||||
// Consume this round of signals: clear pending and handle reset/concurrency
|
||||
_ = Interlocked.Exchange(ref _pending, 0);
|
||||
|
||||
_evt.Reset();
|
||||
|
||||
if (Volatile.Read(ref _pending) != 0) {
|
||||
_evt.Set();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="ThreadHelper"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.ProcessorAbstraction;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Threading
|
||||
{
|
||||
public static partial class ThreadHelper
|
||||
{
|
||||
|
||||
private static readonly int mainThreadId = Environment.CurrentManagedThreadId;
|
||||
|
||||
private static readonly ThreadLocal<bool> isNotMainThread = new(
|
||||
() => Environment.CurrentManagedThreadId != mainThreadId);
|
||||
|
||||
static ThreadHelper()
|
||||
{
|
||||
EmptyThread.Start();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An individual thread doing nothing, and its <see cref="Thread.IsAlive" /> <see
|
||||
/// langword="property" /> always returns false.
|
||||
/// </summary>
|
||||
public static Thread EmptyThread { get; } = new(() => { });
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public static void BindToLogicalCore(
|
||||
LogicalCore core
|
||||
)
|
||||
{
|
||||
SetThreadAffinity(core.GlobalCpuId, core.NumaNodeId);
|
||||
}
|
||||
|
||||
[Conditional("DEBUG")]
|
||||
internal static void MainThreadAssert()
|
||||
{
|
||||
if (isNotMainThread.Value) {
|
||||
throw new InvalidOperationException(
|
||||
$"Attempt to call sensitive operation which can be only called from main thread from thread {Environment.CurrentManagedThreadId}.");
|
||||
}
|
||||
}
|
||||
|
||||
#pragma warning disable CA5393
|
||||
|
||||
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
|
||||
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])]
|
||||
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
|
||||
private static partial void SetThreadAffinity(
|
||||
uint globalCoreId,
|
||||
uint numaNodeId
|
||||
);
|
||||
|
||||
#pragma warning restore CA5393
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="VariableCapacitySemaphore"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Threading
|
||||
{
|
||||
/// <summary>
|
||||
/// A counting semaphore with adjustable maximum capacity at runtime. Implemented using one or
|
||||
/// more ManualResetEventSlim (MRES) gates and atomic counters to avoid kernel semaphore
|
||||
/// overhead while supporting high-frequency Release/Wait patterns. Notes: - This is a
|
||||
/// counting primitive (not a barrier). Permits accumulate up to <see cref="MaxPermits" />. -
|
||||
/// Reducing <see cref="MaxPermits" /> does not revoke already issued permits; it only clamps
|
||||
/// future releases. - Striped MRES can reduce contention in highly parallel waits.
|
||||
/// </summary>
|
||||
public sealed class VariableCapacitySemaphore : IDisposable
|
||||
{
|
||||
|
||||
// MRES stripes (one gate per stripe). All stripes are set when transitioning 0 -> >0 and reset at >0 -> 0.
|
||||
private readonly ManualResetEventSlim[] _events;
|
||||
private volatile int _current;
|
||||
private volatile int _max;
|
||||
private readonly int _stripeMask; // for power-of-two stripe counts; otherwise -1
|
||||
|
||||
/// <summary>
|
||||
/// Create a new variable-capacity semaphore.
|
||||
/// </summary>
|
||||
/// <param name="initialCount">
|
||||
/// Initial available permits (clamped to [0, maxPermits]).
|
||||
/// </param>
|
||||
/// <param name="maxPermits">
|
||||
/// Logical maximum permits (>= 1).
|
||||
/// </param>
|
||||
/// <param name="stripes">
|
||||
/// Optional stripe count for MRES (<= 0 to auto choose by CPU).
|
||||
/// </param>
|
||||
public VariableCapacitySemaphore(
|
||||
int initialCount = 0,
|
||||
int maxPermits = 1,
|
||||
int stripes = 0
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(maxPermits, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(initialCount);
|
||||
|
||||
_max = maxPermits;
|
||||
_current = Math.Min(initialCount, maxPermits);
|
||||
|
||||
int stripeCount = (stripes > 0) ?
|
||||
stripes :
|
||||
ClampToPow2(Math.Min(Environment.ProcessorCount, 8));
|
||||
_events = new ManualResetEventSlim[stripeCount];
|
||||
bool set = _current > 0;
|
||||
for (int i = 0; i < stripeCount; i++) {
|
||||
_events[i] = new ManualResetEventSlim(set, 0);
|
||||
}
|
||||
|
||||
_stripeMask = IsPow2(stripeCount) ? (stripeCount - 1) : (-1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Current number of available permits.
|
||||
/// </summary>
|
||||
public int CurrentCount => Volatile.Read(ref _current);
|
||||
|
||||
/// <summary>
|
||||
/// Logical maximum number of permits. Can be changed at runtime.
|
||||
/// </summary>
|
||||
public int MaxPermits
|
||||
{
|
||||
get => Volatile.Read(ref _max);
|
||||
set
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(value, 1);
|
||||
|
||||
Volatile.Write(ref _max, value);
|
||||
|
||||
// Clamp current to not exceed the new max (do not block; let Wait() consume down).
|
||||
while (true)
|
||||
{
|
||||
int cur = Volatile.Read(ref _current);
|
||||
if (cur <= value)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _current, value, cur) == cur)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Reflect availability to gates
|
||||
if (Volatile.Read(ref _current) > 0)
|
||||
{
|
||||
SetAll();
|
||||
} else
|
||||
{
|
||||
ResetAll();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (ManualResetEventSlim e in _events) {
|
||||
e.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Release a single permit if under <see cref="MaxPermits" />. Returns the previous
|
||||
/// count.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Release()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
int cur = Volatile.Read(ref _current);
|
||||
int max = Volatile.Read(ref _max);
|
||||
if (cur >= max) {
|
||||
return cur;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _current, cur + 1, cur) == cur)
|
||||
{
|
||||
if (cur == 0) {
|
||||
SetAll();
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Release multiple permits (capped by <see cref="MaxPermits" />). Returns the previous
|
||||
/// count.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public int Release(
|
||||
int permits
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(permits);
|
||||
while (true)
|
||||
{
|
||||
int cur = Volatile.Read(ref _current);
|
||||
int max = Volatile.Read(ref _max);
|
||||
if (cur >= max) {
|
||||
return cur;
|
||||
}
|
||||
|
||||
long target = ((long)cur) + permits;
|
||||
if (target > max) {
|
||||
target = max;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _current, (int)target, cur) == cur)
|
||||
{
|
||||
if (cur == 0) {
|
||||
SetAll();
|
||||
}
|
||||
|
||||
return cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Try to take a permit without blocking. Returns true if acquired.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool TryWait()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
int cur = Volatile.Read(ref _current);
|
||||
if (cur == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur)
|
||||
{
|
||||
if (cur - 1 == 0) {
|
||||
ResetAll();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Block until a permit is available, then consume one.
|
||||
/// </summary>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public void Wait()
|
||||
{
|
||||
ManualResetEventSlim gate = GetStripeEvent();
|
||||
SpinWait spinner = new SpinWait();
|
||||
while (true)
|
||||
{
|
||||
int cur = Volatile.Read(ref _current);
|
||||
if ((cur > 0) && (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur))
|
||||
{
|
||||
if (cur - 1 == 0) {
|
||||
ResetAll();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
if (!spinner.NextSpinWillYield)
|
||||
{
|
||||
spinner.SpinOnce();
|
||||
} else
|
||||
{
|
||||
gate.Wait();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Block until a permit is available or canceled, then consume one.
|
||||
/// </summary>
|
||||
public bool Wait(
|
||||
int millisecondsTimeout,
|
||||
CancellationToken cancellationToken = default
|
||||
)
|
||||
{
|
||||
ManualResetEventSlim gate = GetStripeEvent();
|
||||
SpinWait spinner = new SpinWait();
|
||||
int start = Environment.TickCount;
|
||||
int remaining = millisecondsTimeout;
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
int cur = Volatile.Read(ref _current);
|
||||
if ((cur > 0) && (Interlocked.CompareExchange(ref _current, cur - 1, cur) == cur))
|
||||
{
|
||||
if (cur - 1 == 0) {
|
||||
ResetAll();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
if (!spinner.NextSpinWillYield)
|
||||
{
|
||||
spinner.SpinOnce();
|
||||
continue;
|
||||
}
|
||||
if (millisecondsTimeout < 0)
|
||||
{
|
||||
gate.Wait(cancellationToken);
|
||||
continue;
|
||||
}
|
||||
if (remaining <= 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!gate.Wait(remaining, cancellationToken)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
remaining = millisecondsTimeout - (Environment.TickCount - start);
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static int ClampToPow2(
|
||||
int x
|
||||
)
|
||||
{
|
||||
if (x < 1) {
|
||||
x = 1;
|
||||
}
|
||||
|
||||
if (x > 8) {
|
||||
x = 8;
|
||||
}
|
||||
|
||||
int p = 1;
|
||||
while (p << 1 <= x) {
|
||||
p <<= 1;
|
||||
}
|
||||
|
||||
return p;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private ManualResetEventSlim GetStripeEvent()
|
||||
{
|
||||
int id = Environment.CurrentManagedThreadId & 0x7fffffff;
|
||||
if (_stripeMask != -1) {
|
||||
return _events[id & _stripeMask];
|
||||
}
|
||||
|
||||
int idx = id % _events.Length;
|
||||
return _events[idx];
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private static bool IsPow2(
|
||||
int x
|
||||
)
|
||||
{
|
||||
return (x & (x - 1)) == 0;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void ResetAll()
|
||||
{
|
||||
foreach (ManualResetEventSlim e in _events) {
|
||||
e.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void SetAll()
|
||||
{
|
||||
foreach (ManualResetEventSlim e in _events) {
|
||||
e.Set();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="StgSharpTime"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Collections;
|
||||
using StgSharp.Timing;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp
|
||||
{
|
||||
public static partial class World
|
||||
{
|
||||
|
||||
public static TimeSourceProviderBase MainTimeProvider => StgSharpTime.OnlyInstance;
|
||||
|
||||
}
|
||||
|
||||
public sealed class StgSharpTime : TimeSourceProviderBase
|
||||
{
|
||||
|
||||
private static readonly LinearBag<TimeSpanProvider> _subscribers = new();
|
||||
private static readonly Stopwatch mainProvider = new();
|
||||
private static Thread timeProvideThread;
|
||||
|
||||
private StgSharpTime() { }
|
||||
|
||||
// Use Stopwatch frequency as base tick frequency (ticks per second).
|
||||
public override long Frequency => Stopwatch.Frequency;
|
||||
|
||||
public static StgSharpTime OnlyInstance { get; } = new StgSharpTime();
|
||||
|
||||
public override long GetCurrentTimeSpanTick()
|
||||
{
|
||||
return mainProvider.ElapsedTicks;
|
||||
}
|
||||
|
||||
public sealed override void StartProvidingTime()
|
||||
{
|
||||
if (mainProvider.IsRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
mainProvider.Start();
|
||||
timeProvideThread = new Thread(ProvideTime)
|
||||
{
|
||||
IsBackground = true,
|
||||
Name = "StgSharpTime"
|
||||
};
|
||||
timeProvideThread.Start();
|
||||
}
|
||||
|
||||
public override void StopProvidingTime()
|
||||
{
|
||||
lock (mainProvider) {
|
||||
mainProvider.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
public void Terminate()
|
||||
{
|
||||
lock (mainProvider) {
|
||||
mainProvider.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
protected sealed override void AddSubscriberUnsynced(
|
||||
TimeSpanProvider subscriber
|
||||
)
|
||||
{
|
||||
_subscribers.Add(subscriber);
|
||||
}
|
||||
|
||||
protected sealed override void RemoveSubscriberUnsynced(
|
||||
TimeSpanProvider subscriber
|
||||
)
|
||||
{
|
||||
_subscribers.Remove(subscriber);
|
||||
}
|
||||
|
||||
private void ProvideTime()
|
||||
{
|
||||
long internalFrequency = Stopwatch.Frequency; // ticks per second
|
||||
lock (mainProvider) {
|
||||
mainProvider.Restart();
|
||||
}
|
||||
|
||||
while (mainProvider.IsRunning)
|
||||
{
|
||||
long elapsedTicks;
|
||||
lock (mainProvider) {
|
||||
elapsedTicks = mainProvider.ElapsedTicks;
|
||||
}
|
||||
if (_subscribers.Count == 0)
|
||||
{
|
||||
Thread.SpinWait(50);
|
||||
continue;
|
||||
}
|
||||
lock (SubscribeLock)
|
||||
{
|
||||
for (int i = 0; i < _subscribers.Count; i++)
|
||||
{
|
||||
if (!_subscribers[i].OnSpanTick(elapsedTicks))
|
||||
{
|
||||
// Console.WriteLine("removed");
|
||||
_subscribers.RemoveAt(i);
|
||||
i--; // adjust index after removal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Optional: brief spin to avoid tight loop if frame lengths are large.
|
||||
Thread.SpinWait(20);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TimeSequence"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Timing;
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace StgSharp.Timing
|
||||
{
|
||||
public abstract class TimeSequence
|
||||
{
|
||||
|
||||
public long BeginningTick { get; protected set; }
|
||||
|
||||
public long EndingTick { get; protected set; }
|
||||
|
||||
public long PreviousFrameTick { get; protected set; }
|
||||
|
||||
public static TimeSequence CreateLinear(
|
||||
double totalSeconds,
|
||||
double frameSeconds
|
||||
)
|
||||
{
|
||||
if (double.IsPositiveInfinity(totalSeconds)) {
|
||||
return new InfiniteLinearTimeSequence(frameSeconds);
|
||||
}
|
||||
return new LinearTimeSequence(totalSeconds, frameSeconds);
|
||||
}
|
||||
|
||||
public static TimeSequence CreateLinearInfinite(
|
||||
double frameSeconds
|
||||
)
|
||||
{
|
||||
return new InfiniteLinearTimeSequence(frameSeconds);
|
||||
}
|
||||
|
||||
public static TimeSequence CreateLogarithmic(
|
||||
double initialSliceSeconds,
|
||||
double totalSeconds,
|
||||
double growthMultiplier,
|
||||
double minSliceSeconds
|
||||
)
|
||||
{
|
||||
return new LogTimeSequence(initialSliceSeconds, totalSeconds, growthMultiplier,
|
||||
minSliceSeconds);
|
||||
}
|
||||
|
||||
public abstract bool EndsAt(
|
||||
long currentTick
|
||||
);
|
||||
|
||||
public abstract bool IsNextFrameReady(
|
||||
long currentTick
|
||||
);
|
||||
|
||||
public abstract void StartFrom(
|
||||
TimeSourceProviderBase source
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
internal sealed class LinearTimeSequence : TimeSequence
|
||||
{
|
||||
|
||||
private readonly double _frameSeconds;
|
||||
private readonly double _totalSeconds;
|
||||
private long _frameLengthTicks;
|
||||
|
||||
public LinearTimeSequence(
|
||||
double totalSeconds,
|
||||
double frameSeconds
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(totalSeconds);
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(frameSeconds);
|
||||
if (frameSeconds > totalSeconds) {
|
||||
throw new ArgumentException("Frame length greater than total length.");
|
||||
}
|
||||
_totalSeconds = totalSeconds;
|
||||
_frameSeconds = frameSeconds;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool EndsAt(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
return currentTick >= EndingTick;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool IsNextFrameReady(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
long span = currentTick - PreviousFrameTick;
|
||||
if (span >= _frameLengthTicks && currentTick < EndingTick)
|
||||
{
|
||||
PreviousFrameTick += _frameLengthTicks;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public override void StartFrom(
|
||||
TimeSourceProviderBase source
|
||||
)
|
||||
{
|
||||
long f = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
|
||||
_frameLengthTicks = (long)Math.Max(1, _frameSeconds * f);
|
||||
long totalTicks = (long)Math.Max(_frameLengthTicks, Math.Round(_totalSeconds * f, MidpointRounding.AwayFromZero));
|
||||
BeginningTick = source.GetCurrentTimeSpanTick();
|
||||
EndingTick = BeginningTick + totalTicks;
|
||||
EndingTick = EndingTick < 0 ? long.MaxValue : EndingTick;
|
||||
PreviousFrameTick = BeginningTick;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logarithmic time sequence with growth factor and minimum slice length enforcement.
|
||||
/// </summary>
|
||||
internal sealed class LogTimeSequence : TimeSequence
|
||||
{
|
||||
|
||||
private bool _finalEmitted;
|
||||
private readonly double _initialSeconds;
|
||||
private double _lastBoundarySecondsAdjusted;
|
||||
private readonly double _minSliceSeconds;
|
||||
private readonly double _multiplier;
|
||||
private double _nextBoundarySecondsAdjusted;
|
||||
private readonly double _totalSeconds;
|
||||
private int _sliceIndex;
|
||||
private long _frequency;
|
||||
private long _nextBoundaryTick;
|
||||
private long _totalTicks;
|
||||
|
||||
public LogTimeSequence(
|
||||
double initialSliceSeconds,
|
||||
double totalSeconds,
|
||||
double growthMultiplier,
|
||||
double minSliceSeconds
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(initialSliceSeconds);
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(totalSeconds);
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(growthMultiplier);
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(minSliceSeconds);
|
||||
if (growthMultiplier < 1.0) {
|
||||
throw new ArgumentException("growthMultiplier must be >= 1");
|
||||
}
|
||||
if (initialSliceSeconds > totalSeconds) {
|
||||
throw new ArgumentException("Initial slice length greater than total length.");
|
||||
}
|
||||
_initialSeconds = initialSliceSeconds;
|
||||
_totalSeconds = totalSeconds;
|
||||
_multiplier = growthMultiplier;
|
||||
_minSliceSeconds = minSliceSeconds;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool EndsAt(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
return currentTick >= EndingTick;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool IsNextFrameReady(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
// Emit final once when reaching or passing end.
|
||||
if (currentTick >= EndingTick)
|
||||
{
|
||||
if (_finalEmitted) {
|
||||
return false;
|
||||
}
|
||||
|
||||
PreviousFrameTick = EndingTick;
|
||||
_finalEmitted = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Not reached end: only when we cross the next boundary tick do we emit.
|
||||
if (currentTick < _nextBoundaryTick)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Advance to this boundary.
|
||||
PreviousFrameTick = _nextBoundaryTick;
|
||||
_lastBoundarySecondsAdjusted = _nextBoundarySecondsAdjusted;
|
||||
_sliceIndex++;
|
||||
|
||||
// Compute next cumulative boundary seconds with multiplier, then enforce min-slice and total cap.
|
||||
double next = _initialSeconds * Math.Pow(_multiplier, _sliceIndex);
|
||||
if (next > _totalSeconds) {
|
||||
next = _totalSeconds;
|
||||
}
|
||||
|
||||
double delta = next - _lastBoundarySecondsAdjusted;
|
||||
if (delta < _minSliceSeconds)
|
||||
{
|
||||
next = _lastBoundarySecondsAdjusted + _minSliceSeconds;
|
||||
if (next > _totalSeconds) {
|
||||
next = _totalSeconds;
|
||||
}
|
||||
}
|
||||
|
||||
_nextBoundarySecondsAdjusted = next;
|
||||
_nextBoundaryTick = (_nextBoundarySecondsAdjusted >= _totalSeconds) ?
|
||||
EndingTick :
|
||||
BeginningTick + SecondsToTicks(_nextBoundarySecondsAdjusted);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public override void StartFrom(
|
||||
TimeSourceProviderBase source
|
||||
)
|
||||
{
|
||||
_frequency = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
|
||||
BeginningTick = source.GetCurrentTimeSpanTick();
|
||||
_totalTicks = SecondsToTicks(_totalSeconds);
|
||||
EndingTick = BeginningTick + _totalTicks;
|
||||
_sliceIndex = 0;
|
||||
_lastBoundarySecondsAdjusted = 0.0;
|
||||
_nextBoundarySecondsAdjusted = _initialSeconds;
|
||||
_nextBoundaryTick = BeginningTick + SecondsToTicks(_nextBoundarySecondsAdjusted);
|
||||
PreviousFrameTick = BeginningTick;
|
||||
_finalEmitted = false;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private long SecondsToTicks(
|
||||
double seconds
|
||||
)
|
||||
{
|
||||
return (long)Math.Max(1L, Math.Round(seconds * _frequency, MidpointRounding.AwayFromZero));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Infinite linear time sequence: emits frames at fixed interval and never ends.
|
||||
/// </summary>
|
||||
internal sealed class InfiniteLinearTimeSequence : TimeSequence
|
||||
{
|
||||
|
||||
private readonly double _frameSeconds;
|
||||
private long _frameLengthTicks;
|
||||
|
||||
public InfiniteLinearTimeSequence(
|
||||
double frameSeconds
|
||||
)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(frameSeconds);
|
||||
_frameSeconds = frameSeconds;
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool EndsAt(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
return false; // never ends
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public override bool IsNextFrameReady(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
long span = currentTick - PreviousFrameTick;
|
||||
if (span >= _frameLengthTicks)
|
||||
{
|
||||
PreviousFrameTick += _frameLengthTicks;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public override void StartFrom(
|
||||
TimeSourceProviderBase source
|
||||
)
|
||||
{
|
||||
long f = source?.Frequency ?? throw new ArgumentNullException(nameof(source));
|
||||
_frameLengthTicks = (long)Math.Max(1, _frameSeconds * f);
|
||||
BeginningTick = source.GetCurrentTimeSpanTick();
|
||||
EndingTick = long.MaxValue; // sentinel
|
||||
PreviousFrameTick = BeginningTick;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TimeSourceProvider"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp;
|
||||
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Timing
|
||||
{
|
||||
public abstract class TimeSourceProviderBase
|
||||
{
|
||||
|
||||
public abstract long Frequency { get; }
|
||||
|
||||
protected object SubscribeLock { get; } = new();
|
||||
|
||||
public void AddSubscriber(
|
||||
TimeSpanProvider subscriber
|
||||
)
|
||||
{
|
||||
lock (SubscribeLock) {
|
||||
AddSubscriberUnsynced(subscriber);
|
||||
}
|
||||
}
|
||||
|
||||
public abstract long GetCurrentTimeSpanTick();
|
||||
|
||||
public void RemoveSubscriber(
|
||||
TimeSpanProvider subscriber
|
||||
)
|
||||
{
|
||||
lock (SubscribeLock) {
|
||||
RemoveSubscriberUnsynced(subscriber);
|
||||
}
|
||||
}
|
||||
|
||||
public abstract void StartProvidingTime();
|
||||
|
||||
public abstract void StopProvidingTime();
|
||||
|
||||
protected abstract void AddSubscriberUnsynced(
|
||||
TimeSpanProvider subscriber
|
||||
);
|
||||
|
||||
protected abstract void RemoveSubscriberUnsynced(
|
||||
TimeSpanProvider subscriber
|
||||
);
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TimeSpanProvider.Awaiting"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.HighPerformance.Memory;
|
||||
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace StgSharp.Timing
|
||||
{
|
||||
public partial class TimeSpanProvider
|
||||
{
|
||||
|
||||
private static readonly ConcurrentBufferStack<int> s_unusedID = new ConcurrentBufferStack<int>(8);
|
||||
|
||||
private static int s_maxID;
|
||||
|
||||
private EventHandler? _refreshHandler, _missHandler;
|
||||
private int _size, _id;
|
||||
private long _pendingSpans; // Number of unconsumed span ticks since last Wait.
|
||||
|
||||
private readonly ManualResetEventSlim _event = new();
|
||||
private readonly object _participantObject = new();
|
||||
|
||||
public event EventHandler MissTimeSpanRefreshed
|
||||
{
|
||||
add => _missHandler += value;
|
||||
remove => _missHandler -= value;
|
||||
}
|
||||
|
||||
public event EventHandler TimeSpanRefreshed
|
||||
{
|
||||
add => _refreshHandler += value;
|
||||
remove => _refreshHandler -= value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True if there is at least one unconsumed span tick (PendingSpanCount > 0).
|
||||
/// </summary>
|
||||
public bool HasPendingSpan => PendingSpanCount > 0;
|
||||
|
||||
public int TokenID
|
||||
{
|
||||
get => _id;
|
||||
private set => _id = value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of span ticks not yet consumed by Wait ( > 0 means last tick not waited).
|
||||
/// </summary>
|
||||
public long PendingSpanCount => Volatile.Read(ref _pendingSpans);
|
||||
|
||||
internal ManualResetEventSlim AwaitingEvent => _event;
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
// mark provider as ended so external users can observe terminal state
|
||||
Volatile.Write(ref _endedFlag, 1);
|
||||
s_unusedID.Push(TokenID);
|
||||
_event.Set();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
public override int GetHashCode()
|
||||
{
|
||||
return _id;
|
||||
}
|
||||
|
||||
public void Participant()
|
||||
{
|
||||
lock (_participantObject)
|
||||
{
|
||||
// Increase the maximum concurrent consumers for the next tick.
|
||||
_size++;
|
||||
}
|
||||
}
|
||||
|
||||
public void WaitNextSpan()
|
||||
{
|
||||
_event.Wait();
|
||||
|
||||
// Consume one span slot (clamp at 0 so multiple subscribers won't make it negative).
|
||||
while (true)
|
||||
{
|
||||
long cur = Volatile.Read(ref _pendingSpans);
|
||||
if (cur == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
|
||||
{
|
||||
if (cur - 1 == 0)
|
||||
{
|
||||
// Last consumer resets the gate for the next tick.
|
||||
_event.Reset();
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async Task WaitNextSpanAsync()
|
||||
{
|
||||
await Task.Run(() => _event.Wait()).ConfigureAwait(false);
|
||||
while (true)
|
||||
{
|
||||
long cur = Volatile.Read(ref _pendingSpans);
|
||||
if (cur == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
|
||||
{
|
||||
if (cur - 1 == 0) {
|
||||
_event.Reset();
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wait for the next timespan tick, but honor the provided cancellation token. Throws
|
||||
/// OperationCanceledException when token is cancelled. If the provider is disposed during
|
||||
/// wait an OperationCanceledException is thrown to indicate end.
|
||||
/// </summary>
|
||||
public Task WaitNextSpanAsync(
|
||||
CancellationToken ct
|
||||
)
|
||||
{
|
||||
// Block the current thread until the event is signaled or cancellation requested.
|
||||
_event.Wait(ct);
|
||||
|
||||
// consume one pending span (same logic as non-cancellable variant)
|
||||
while (true)
|
||||
{
|
||||
long cur = Volatile.Read(ref _pendingSpans);
|
||||
if (cur == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (Interlocked.CompareExchange(ref _pendingSpans, cur - 1, cur) == cur)
|
||||
{
|
||||
if (cur - 1 == 0) {
|
||||
_event.Reset();
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
internal void MissRefresh()
|
||||
{
|
||||
_missHandler?.Invoke(this, EventArgs.Empty);
|
||||
} // Legacy compatibility.
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
/// <param name="currentTick">
|
||||
/// Tick passed since binded time source.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if the span provider is working. False if time is over (the sequence ended).
|
||||
/// </returns>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
internal bool OnSpanTick(
|
||||
long currentTick
|
||||
)
|
||||
{
|
||||
if (_sequence.EndsAt(currentTick))
|
||||
{
|
||||
// mark ended due to sequence timeout, then dispose
|
||||
Volatile.Write(ref _endedFlag, 1);
|
||||
|
||||
// Console.WriteLine("Time span ended.");
|
||||
Dispose();
|
||||
return false;
|
||||
}
|
||||
if (!_sequence.IsNextFrameReady(currentTick)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Console.WriteLine("Frame hit!");
|
||||
|
||||
// Fast path: if no pending consumers, initialize the counter to current capacity (_size).
|
||||
if (Volatile.Read(ref _pendingSpans) == 0)
|
||||
{
|
||||
int toIssue = Math.Max(1, Volatile.Read(ref _size));
|
||||
if (Interlocked.CompareExchange(ref _pendingSpans, toIssue, 0) == 0)
|
||||
{
|
||||
// Open the gate for all current waiters (up to `_size`).
|
||||
_event.Set();
|
||||
_refreshHandler?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
} else
|
||||
{
|
||||
// A tick is already pending (or was just set by another thread): mark as miss.
|
||||
_missHandler?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
~TimeSpanProvider()
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
// -----------------------------------------------------------------------------
|
||||
// file="TimeSpanProvider"
|
||||
// Project: StgSharp
|
||||
// Copyright (c) Nitload.
|
||||
// SPDX-License-Identifier: MIT
|
||||
// -----------------------------------------------------------------------------
|
||||
|
||||
using StgSharp.Timing;
|
||||
using System;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace StgSharp.Timing
|
||||
{
|
||||
/// <summary>
|
||||
/// TimeSpanProvider: uses a bound TimeSequence and TimeSourceProviderBase to publish span
|
||||
/// ticks. Each OnSpanTick advances the sequence; when sequence ends provider unsubscribes.
|
||||
/// </summary>
|
||||
public partial class TimeSpanProvider
|
||||
{
|
||||
|
||||
// private bool _subscribed;
|
||||
private readonly int _emittedSpanCount; // legacy counter
|
||||
|
||||
// internal ended flag (0 = running, 1 = ended)
|
||||
private int _endedFlag;
|
||||
private readonly TimeSequence _sequence;
|
||||
private readonly TimeSourceProviderBase _timeSource;
|
||||
|
||||
public TimeSpanProvider(
|
||||
TimeSequence sequence,
|
||||
TimeSourceProviderBase provider,
|
||||
int maxWaitCount = 1
|
||||
)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(provider);
|
||||
ArgumentNullException.ThrowIfNull(sequence);
|
||||
_timeSource = provider;
|
||||
_sequence = sequence;
|
||||
_sequence.StartFrom(provider);
|
||||
provider.AddSubscriber(this);
|
||||
|
||||
// _subscribed = true;
|
||||
if (!s_unusedID.TryPop(out int id))
|
||||
{
|
||||
id = Interlocked.Increment(ref s_maxID);
|
||||
}
|
||||
TokenID = id;
|
||||
_size = maxWaitCount > 0 ? maxWaitCount : 1;
|
||||
_event = new ManualResetEventSlim(initialState:false, spinCount:0);
|
||||
_pendingSpans = 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the provider has reached its end and will not emit further spans.
|
||||
/// </summary>
|
||||
public bool IsEnded => Volatile.Read(ref _endedFlag) != 0;
|
||||
|
||||
/// <summary>
|
||||
/// Current emitted time in seconds relative to the sequence beginning.
|
||||
/// </summary>
|
||||
public double CurrentSecond => (double)(_sequence.PreviousFrameTick - _sequence.BeginningTick) / _timeSource.Frequency;
|
||||
|
||||
public int CurrentSpan => _emittedSpanCount;
|
||||
|
||||
public static TimeSourceProviderBase DefaultProvider => World.MainTimeProvider;
|
||||
|
||||
public static int ToInt32(
|
||||
TimeSpanProvider provider
|
||||
)
|
||||
{
|
||||
return provider?._emittedSpanCount ?? 0;
|
||||
}
|
||||
|
||||
public static explicit operator int(
|
||||
TimeSpanProvider provider
|
||||
)
|
||||
{
|
||||
return provider?._emittedSpanCount ?? 0;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user