This commit is contained in:
Youmu Liu
2026-03-26 18:02:13 +08:00
parent d960d02972
commit 06e3aa119b
20 changed files with 198 additions and 169 deletions
@@ -33,9 +33,11 @@ using System.Threading.Tasks;
namespace StgSharp.HighPerformance.Memory namespace StgSharp.HighPerformance.Memory
{ {
public interface IPrefetchPredict public interface IL4Predict
{ {
bool IsExhausted { get; }
/// <summary> /// <summary>
/// Writes predicted cache line descriptors into <paramref name="node" />. /// Writes predicted cache line descriptors into <paramref name="node" />.
/// </summary> /// </summary>
@@ -43,17 +45,20 @@ namespace StgSharp.HighPerformance.Memory
/// Prediction output buffer. /// Prediction output buffer.
/// </param> /// </param>
/// <param name="count"> /// <param name="count">
/// When this method returns, contains the number of valid entries written to <paramref /// When this method returns, contains the number of valid entries written to <paramref ///
/// name="node" />. The value must not exceed <paramref name="node" />.Length. /// name="node" />. The value must not exceed <paramref name="node" />.Length.
/// </param> /// </param>
/// <returns> /// <returns>
/// <see langword="true" /> if any prediction data was produced; otherwise, <see langword="false" /// <see langword="true" /> if any prediction data was produced; otherwise, <see langword="false"
/// />. /// />.
/// </returns> /// </returns>
bool Predict(Span<CacheLinePrediction> node, out int count); bool Predict(
Span<CacheLinePrediction> node,
out int count
);
/// <summary> /// <summary>
/// Materializes the cache line identified by <paramref name="origin" /> into <paramref /// Materializes the cache line identified by <paramref name="origin" /> into <paramref ///
/// name="cache" />. /// name="cache" />.
/// </summary> /// </summary>
/// <param name="origin"> /// <param name="origin">
@@ -65,7 +70,11 @@ namespace StgSharp.HighPerformance.Memory
/// <param name="cache"> /// <param name="cache">
/// Destination cache line buffer. /// Destination cache line buffer.
/// </param> /// </param>
void Prefetch(nuint origin, ulong policy, Span<byte> cache); void Prefetch(
nuint origin,
ulong policy,
Span<byte> cache
);
/// <summary> /// <summary>
/// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />. /// Writes the cache line in <paramref name="cache" /> back to <paramref name="origin" />.
@@ -79,21 +88,29 @@ namespace StgSharp.HighPerformance.Memory
/// <param name="cache"> /// <param name="cache">
/// Source cache line buffer. /// Source cache line buffer.
/// </param> /// </param>
void WriteBack(nuint origin, ulong policy, ReadOnlySpan<byte> cache); void WriteBack(
nuint origin,
ulong policy,
ReadOnlySpan<byte> cache
);
} }
public struct CacheLinePrediction : IEquatable<CacheLinePrediction> public struct CacheLinePrediction : IEquatable<CacheLinePrediction>
{ {
public override readonly bool Equals(object? obj) public override readonly bool Equals(
object? obj
)
{ {
return obj is CacheLinePrediction prediction && return obj is CacheLinePrediction prediction &&
Address.Equals(prediction.Address) && Address.Equals(prediction.Address) &&
MapPolicy == prediction.MapPolicy; MapPolicy == prediction.MapPolicy;
} }
public readonly bool Equals(CacheLinePrediction other) public readonly bool Equals(
CacheLinePrediction other
)
{ {
return return
Address == other.Address && MapPolicy == other.MapPolicy; Address == other.Address && MapPolicy == other.MapPolicy;
@@ -104,11 +121,17 @@ namespace StgSharp.HighPerformance.Memory
return HashCode.Combine(Address, MapPolicy); return HashCode.Combine(Address, MapPolicy);
} }
public static bool operator !=(CacheLinePrediction left, CacheLinePrediction right) public static bool operator !=(
CacheLinePrediction left,
CacheLinePrediction right
)
{ {
return !(left == right); return !(left == right);
} }
public static bool operator ==(CacheLinePrediction left, CacheLinePrediction right) public static bool operator ==(
CacheLinePrediction left,
CacheLinePrediction right
)
{ {
return left.Equals(right); return left.Equals(right);
} }
@@ -43,7 +43,9 @@ namespace StgSharp.HighPerformance.Memory
throw new OverflowException("Too many activated cache line in external reference."); throw new OverflowException("Too many activated cache line in external reference.");
} }
Span<CacheLinePrediction> span = new(_prediction, PredictionCount); Span<CacheLinePrediction> span = new(_prediction, PredictionCount);
if (!(PrefetchPredict.Predict(span, out int actualCount) && actualCount > 0)) { if (!PrefetchPredict.Predict(span, out int actualCount) && (actualCount <= 0))
{
_exhausted = true;
return; return;
} }
int missIndex = 0; int missIndex = 0;
@@ -85,7 +87,7 @@ namespace StgSharp.HighPerformance.Memory
head->_origin = address; head->_origin = address;
head->_profile = handle.MapPolicy; head->_profile = handle.MapPolicy;
head->_refCount = 0; head->_refCount = 0;
head->_address = (nuint)(&_data[index]); head->_address = nuint & _data[index];
// pre fetch them here // pre fetch them here
PrefetchPredict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>())); PrefetchPredict.Prefetch(head->_origin, head->_profile, new Span<byte>((byte*)head->_address, Unsafe.SizeOf<CacheLineData>()));
+54 -45
View File
@@ -39,7 +39,6 @@ namespace StgSharp.HighPerformance.Memory
private const int PredictionLeastAhead = 1024; private const int PredictionLeastAhead = 1024;
private readonly CacheLineData* _data; private readonly CacheLineData* _data;
private readonly EvictionArray* _eviction; private readonly EvictionArray* _eviction;
private readonly CacheLineHead* _head; private readonly CacheLineHead* _head;
private readonly CacheLinePrediction* _prediction; private readonly CacheLinePrediction* _prediction;
@@ -57,12 +56,12 @@ namespace StgSharp.HighPerformance.Memory
public L4() public L4()
{ {
// init order is fallow by align requirement of each component. // init order is fallow by align requirement of each component.
nuint cacheSize = (nuint)Unsafe.SizeOf<CacheLineData>() * CacheLineCount; // SIMDID, no less than 16 byte bit nuint cacheSize = ((nuint)Unsafe.SizeOf<CacheLineData>()) * CacheLineCount; // SIMDID, no less than 16 byte bit
nuint entrySize = (nuint)Unsafe.SizeOf<CacheLineHead>() * CacheLineCount; // 16 byte align nuint entrySize = ((nuint)Unsafe.SizeOf<CacheLineHead>()) * CacheLineCount; // 16 byte align
nuint predictionSize = (nuint)Unsafe.SizeOf<CacheLinePrediction>() * PredictionCount; //16 byte align nuint predictionSize = ((nuint)Unsafe.SizeOf<CacheLinePrediction>()) * PredictionCount; //16 byte align
nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align
nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align nuint rrpvSize = (nuint)Unsafe.SizeOf<EvictionArray>(); // 16 byte align
nuint stackSize = (nuint)sizeof(int) * CacheLineCount; // int align nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align
// malloc all memory at once, and free at once in Dispose. // malloc all memory at once, and free at once in Dispose.
// This is to avoid fragmentation and improve performance. // This is to avoid fragmentation and improve performance.
@@ -85,7 +84,17 @@ namespace StgSharp.HighPerformance.Memory
ptr += stackSize; ptr += stackSize;
} }
public IPrefetchPredict PrefetchPredict { get; set; } public IL4Predict PrefetchPredict { get; set; }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void ContinueOnce(
L4CacheLine line
)
{
if (line.Address > 0) {
_ = Interlocked.Decrement(ref _aheadCount);
}
}
[System.Diagnostics.CodeAnalysis.SuppressMessage( [System.Diagnostics.CodeAnalysis.SuppressMessage(
"Design", "Design",
@@ -108,11 +117,11 @@ namespace StgSharp.HighPerformance.Memory
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
public CacheLine Map( public L4CacheLine Map(
nuint origin nuint origin
) )
{ {
if (_aheadCount < PredictionLeastAhead) if ((_aheadCount < PredictionLeastAhead) && !PrefetchPredict.IsExhausted)
{ {
if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0)
{ {
@@ -141,7 +150,7 @@ namespace StgSharp.HighPerformance.Memory
(*_eviction)[entry->_id] = 0; (*_eviction)[entry->_id] = 0;
_ = Interlocked.Increment(ref _activeCount); _ = Interlocked.Increment(ref _activeCount);
_ = Interlocked.Decrement(ref _aheadCount); _ = Interlocked.Decrement(ref _aheadCount);
return new CacheLine(entry->_address, ref _activeCount, ref entry->_refCount); return new L4CacheLine(entry->_address, ref _activeCount, ref entry->_refCount);
} else } else
{ {
_ = Interlocked.Decrement(ref entry->_refCount); _ = Interlocked.Decrement(ref entry->_refCount);
@@ -157,41 +166,6 @@ namespace StgSharp.HighPerformance.Memory
nuint origin nuint origin
); );
public ref struct CacheLine : IDisposable
{
private ref int _activeCount;
private ref int _refCount;
private nuint _handle;
internal CacheLine(
nuint handle,
ref int activeCount,
ref int cacheRefCount
)
{
_handle = handle;
_refCount = ref cacheRefCount;
_activeCount = ref activeCount;
}
[Obsolete("You should not create a CacheLineHandle outside a L4", true)]
public CacheLine() { }
public nuint Address => _handle;
public void Dispose()
{
if (_handle == 0) {
return;
}
_handle = 0;
_ = Interlocked.Decrement(ref _activeCount);
_ = Interlocked.Decrement(ref _refCount);
}
}
private struct CacheLineData private struct CacheLineData
{ {
@@ -200,4 +174,39 @@ namespace StgSharp.HighPerformance.Memory
} }
} }
public ref struct L4CacheLine : IDisposable
{
private ref int _activeCount;
private ref int _refCount;
private nuint _handle;
internal L4CacheLine(
nuint handle,
ref int activeCount,
ref int cacheRefCount
)
{
_handle = handle;
_refCount = ref cacheRefCount;
_activeCount = ref activeCount;
}
[Obsolete("You should not create a CacheLineHandle outside a L4", true)]
public L4CacheLine() { }
public readonly nuint Address => _handle;
public void Dispose()
{
if (_handle == 0) {
return;
}
_handle = 0;
_ = Interlocked.Decrement(ref _activeCount);
_ = Interlocked.Decrement(ref _refCount);
}
}
} }
+15 -15
View File
@@ -64,7 +64,10 @@ namespace StgSharp.Mathematics.Numeric
{ {
get get
{ {
if (y < 0 || y > _size.ElementColumnLength || x < 0 || x > _size.ElementRowLength) { if ((y < 0) ||
(y > _size.ElementColumnLength) ||
(x < 0) ||
(x > _size.ElementRowLength)) {
throw new ArgumentOutOfRangeException($"Index [{x},{y}] is out of matrix index range."); throw new ArgumentOutOfRangeException($"Index [{x},{y}] is out of matrix index range.");
} }
long offset = GetElementOffset(x, y); long offset = GetElementOffset(x, y);
@@ -75,7 +78,10 @@ namespace StgSharp.Mathematics.Numeric
} }
set set
{ {
if (y < 0 || y > _size.ElementColumnLength || x < 0 || x > _size.ElementRowLength) { if ((y < 0) ||
(y > _size.ElementColumnLength) ||
(x < 0) ||
(x > _size.ElementRowLength)) {
throw new ArgumentOutOfRangeException($"Index [{x},{y}] is out of matrix index range."); throw new ArgumentOutOfRangeException($"Index [{x},{y}] is out of matrix index range.");
} }
long offset = GetElementOffset(x, y); long offset = GetElementOffset(x, y);
@@ -106,8 +112,8 @@ namespace StgSharp.Mathematics.Numeric
ReadOnlySpan<MatrixIndex> row ReadOnlySpan<MatrixIndex> row
] ]
{ {
get { throw new NotImplementedException(); } get => throw new NotImplementedException();
set { throw new NotImplementedException(); } set => throw new NotImplementedException();
} }
/// <summary> /// <summary>
@@ -122,11 +128,7 @@ namespace StgSharp.Mathematics.Numeric
public MatrixLayout Layout { get; init; } public MatrixLayout Layout { get; init; }
internal MatrixKernel<T>* Buffer internal MatrixKernel<T>* Buffer => (MatrixKernel<T>*)_buffer.BufferPointer;
{
[MethodImpl(MethodImplOptions.AggressiveInlining)]
get => (MatrixKernel<T>*)_buffer.BufferPointer;
}
protected internal int KernelColumnLength => _size.KernelColumnLength; protected internal int KernelColumnLength => _size.KernelColumnLength;
@@ -138,9 +140,9 @@ namespace StgSharp.Mathematics.Numeric
int y int y
) )
{ {
return ref _boarder.IsInBuffer(x, y, in _size) ? return ref (_boarder.IsInBuffer(x, y, in _size) ?
ref _buffer[_boarder.GetIndexOffset(x, y, in _size)] : ref _buffer[_boarder.GetIndexOffset(x, y, in _size)] :
ref Unsafe.NullRef<MatrixKernel<T>>(); ref Unsafe.NullRef<MatrixKernel<T>>());
} }
internal long GetElementOffset( internal long GetElementOffset(
@@ -158,7 +160,7 @@ namespace StgSharp.Mathematics.Numeric
} }
long kernelOffset = _boarder.GetIndexOffset(k_r, k_c, in _size); long kernelOffset = _boarder.GetIndexOffset(k_r, k_c, in _size);
return kernelOffset * 16 + 4 * r + c; return (kernelOffset * 16) + (4 * r) + c;
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -195,8 +197,6 @@ namespace StgSharp.Mathematics.Numeric
// Caller guarantees: indices in range, layout is dense square. // Caller guarantees: indices in range, layout is dense square.
int k_c = y / 4; int k_c = y / 4;
int k_r = x / 4; int k_r = x / 4;
int c = y % 4;
int r = x % 4;
return ref _buffer[_boarder.GetIndexOffset(k_c, k_r, in _size)]; return ref _buffer[_boarder.GetIndexOffset(k_c, k_r, in _size)];
} }
@@ -30,6 +30,7 @@ using StgSharp.Mathematics.Numeric.Runtime;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Runtime.CompilerServices;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -71,34 +72,45 @@ namespace StgSharp.Mathematics.Numeric.Linear
t->Mat1 = (MatrixKernel*)leftPtr; t->Mat1 = (MatrixKernel*)leftPtr;
t->Mat2 = (MatrixKernel*)rightPtr; t->Mat2 = (MatrixKernel*)rightPtr;
t->Mat3 = (MatrixKernel*)ansPtr; t->Mat3 = (MatrixKernel*)ansPtr;
t->ElementType = MatrixElementType.F64; t->ElementType = f32;
t->ComputeHandle = MatrixIntrinsicHandle.Add;
t->Scalar.Data<long>(0) = count; t->Scalar.Data<long>(0) = count;
SequentialOperation(t, count);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void SequentialOperation(
MatrixParallelTask* task,
long count
)
{
MatrixElementType elementType = task->ElementType;
MatrixParallelThread[] pool; MatrixParallelThread[] pool;
MkProc op = (MkProc)NumericalModule.GlobalContext.
mat_mk[elementType.IntrinsicNode].
Get(task->ComputeHandle);
if (count <= 256) if (count <= 256)
{ {
add(t); op(task);
// .buffer_add(leftPtr, rightPtr, ansPtr, count);
return; return;
} else if (count <= 1024) } else if (count <= 1024)
{ {
pool = MatrixParallel.LeadParallel(1); pool = MatrixParallel.LeadParallel(1);
add(t); op(task);
MatrixParallel.ReturnParallelResources(ref pool); MatrixParallel.ReturnParallelResources(ref pool);
return; return;
} }
pool = MatrixParallel.LeadParallel((int)(count / 1024)); pool = MatrixParallel.LeadParallel(count / 1024);
MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask<SequentialPredictor>(pool, t); MatrixParallelWrap taskGroup = MatrixParallel.ScheduleTask<SequentialPredictor>(pool, task);
taskGroup.LaunchParallel(SleepMode.DeepSleep); taskGroup.LaunchParallel(SleepMode.DeepSleep);
taskGroup.Dispose(); taskGroup.Dispose();
return;
} }
private struct SequentialPredictor : IMatrixPredictor<SequentialPredictor> private struct SequentialPredictor : IMatrixPredictor<SequentialPredictor>
{ {
public static PredictPolicy Policy => PredictPolicy.Sequential;
public bool Predict( public bool Predict(
Span<CacheLinePrediction> node, Span<CacheLinePrediction> node,
out int count out int count
@@ -45,7 +45,7 @@ namespace StgSharp.Mathematics.Numeric
private class AwaitingQueue private class AwaitingQueue
{ {
private Element _quickElements = new(); private Element _quickElements;
private volatile int _count; private volatile int _count;
private readonly Queue<WaitHandle> _fallback = []; private readonly Queue<WaitHandle> _fallback = [];
@@ -51,7 +51,7 @@ namespace StgSharp.Mathematics.Numeric
internal static unsafe MatrixParallelWrap ScheduleTask<TTask>( internal static unsafe MatrixParallelWrap ScheduleTask<TTask>(
MatrixParallelThread[] workers, MatrixParallelThread[] workers,
MatrixParallelTask* package MatrixParallelTask* package
) where TTask : IPrefetchPredict ) where TTask : IL4Predict
{ {
/* /*
int primCount = package->PrimCount; int primCount = package->PrimCount;
@@ -1,17 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric.Runtime
{
internal class MatrixParallel
{
public MatrixParallelTask BaseTask { get; set; }
public
}
}
@@ -34,6 +34,16 @@ using System.Threading.Tasks;
namespace StgSharp.Mathematics.Numeric.Runtime namespace StgSharp.Mathematics.Numeric.Runtime
{ {
internal interface IMatrixPredictor<TSelf> : IPrefetchPredict internal interface IMatrixPredictor<TSelf> : IL4Predict
where TSelf : unmanaged, IMatrixPredictor<TSelf> { } where TSelf : unmanaged, IMatrixPredictor<TSelf>
{
static abstract PredictPolicy Policy { get; }
}
internal enum PredictPolicy
{
Transparent = 0,
Sequential = 1,
}
} }
@@ -43,22 +43,23 @@ namespace StgSharp.Mathematics.Numeric
{ {
private static readonly ConcurrentStack<MatrixParallelQueue> _recycle = []; private static readonly ConcurrentStack<MatrixParallelQueue> _recycle = [];
private ulong _cursor; private long _cursor;
private ulong _maxCount; private long _maxCount;
private MatrixParallelQueue() { } private MatrixParallelQueue() { }
public MatrixParallelTask* BaseTask { get; private set; } public MatrixParallelTask* BaseTask { get; private set; }
public static MatrixParallelQueue Rent( public static MatrixParallelQueue Rent(
MatrixParallelTask* pack MatrixParallelTask* pack,
long count
) )
{ {
if (!_recycle.TryPop(out MatrixParallelQueue? queue)) { if (!_recycle.TryPop(out MatrixParallelQueue? queue)) {
queue = new MatrixParallelQueue(); queue = new MatrixParallelQueue();
} }
queue.BaseTask = pack; queue.BaseTask = pack;
queue._maxCount = ((ulong)pack->CommonX) * ((ulong)pack->CommonY); queue._maxCount = count;
queue._cursor = 0; queue._cursor = 0;
return queue; return queue;
} }
@@ -49,7 +49,8 @@ namespace StgSharp.Mathematics.Numeric
{ {
private IntrinsicContext* _context; private IntrinsicContext* _context;
private volatile bool _disposed, _isRetireRequested = false; private MatrixParallelTask* _currentTask;
private volatile bool _disposed, _isRetireRequested ;
private readonly ManualResetEventSlim _resetEvent; private readonly ManualResetEventSlim _resetEvent;
private readonly Thread _managedThread; private readonly Thread _managedThread;
@@ -67,8 +68,6 @@ namespace StgSharp.Mathematics.Numeric
public ThreadPriority AfterWork { get; set; } public ThreadPriority AfterWork { get; set; }
internal ManualResetEventSlim? LeaderEvent { get; set; }
internal ref readonly IntrinsicContext Intrinsic => ref (*_context); internal ref readonly IntrinsicContext Intrinsic => ref (*_context);
internal bool IsSingle { get; set; } internal bool IsSingle { get; set; }
@@ -91,6 +90,16 @@ namespace StgSharp.Mathematics.Numeric
_resetEvent.Set(); _resetEvent.Set();
} }
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal void SetSingleThreadTask(
ref MatrixParallelTask task
)
{
if (IsSingle) {
*_currentTask = task;
}
}
protected virtual void Dispose( protected virtual void Dispose(
bool disposing bool disposing
) )
@@ -117,6 +126,7 @@ namespace StgSharp.Mathematics.Numeric
_ = Thread.Yield(); _ = Thread.Yield();
SIMDID feature = new(GetSimdId()); SIMDID feature = new(GetSimdId());
IntrinsicContext* contextPtr = stackalloc IntrinsicContext[1]; IntrinsicContext* contextPtr = stackalloc IntrinsicContext[1];
MatrixParallelTask* taskPtr = stackalloc MatrixParallelTask[4];
LoadIntrinsicFunction(contextPtr, feature.Mask); LoadIntrinsicFunction(contextPtr, feature.Mask);
_context = contextPtr; _context = contextPtr;
State = MatrixThreadState.Ready; State = MatrixThreadState.Ready;
@@ -134,11 +144,11 @@ namespace StgSharp.Mathematics.Numeric
_managedThread.Priority = ThreadPriority.Highest; _managedThread.Priority = ThreadPriority.Highest;
if (IsSingle) if (IsSingle)
{ {
// TODO 将外部线程绑定到当前线程上,并执行任务包 MatrixParallelTask* outer = _currentTask;
IsSingle = false;
} else } else
{ {
MatrixParallelQueue taskQueue = BindedWrap.TaskQueue; MatrixParallelQueue taskQueue = BindedWrap.TaskQueue;
MatrixParallelTask* p = taskQueue.BaseTask;
/* /*
if (0 > (int)p ->ComputeHandle) { if (0 > (int)p ->ComputeHandle) {
SpecialBufferCompute(p); SpecialBufferCompute(p);
@@ -162,19 +172,19 @@ namespace StgSharp.Mathematics.Numeric
#pragma warning disable CA5393 #pragma warning disable CA5393
[LibraryImport(Native.LibName, EntryPoint = "sn_get_simd_level_local")] [LibraryImport(Native.LibName, EntryPoint = "sn_get_simd_level_local")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial ulong GetSimdId(); internal static partial ulong GetSimdId();
[LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")] [LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
internal static partial void SetThreadAffinity( internal static partial void SetThreadAffinity(
int coreId int coreId
); );
[LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] [LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")]
[UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])]
[DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)]
private static unsafe partial void LoadIntrinsicFunction( private static unsafe partial void LoadIntrinsicFunction(
IntrinsicContext* context, IntrinsicContext* context,
@@ -26,6 +26,7 @@
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
using StgSharp.Collections; using StgSharp.Collections;
using StgSharp.HighPerformance.Memory;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel.DataAnnotations; using System.ComponentModel.DataAnnotations;
@@ -59,25 +60,16 @@ namespace StgSharp.Mathematics.Numeric
) )
{ {
_threads = threadArray; _threads = threadArray;
IsMultithread = threadArray.Length > 1;
_countdownEvent = new CountdownEvent(threadArray.Length); _countdownEvent = new CountdownEvent(threadArray.Length);
} }
public ReadOnlySpan<MatrixParallelThread> Threads public ReadOnlySpan<MatrixParallelThread> Threads => _disposed ? [] : _threads;
{
get
{
ObjectDisposedException.ThrowIf(_disposed, nameof(MatrixParallelWrap));
return _threads;
}
}
public ThreadPriority AfterWork { get; private set; } public ThreadPriority AfterWork { get; private set; }
internal MatrixParallelQueue TaskQueue { get; set; } internal L4 DataCache { get; set; }
private bool IsMultithread { get; init; } internal MatrixParallelQueue TaskQueue { get; set; }
public void Dispose() public void Dispose()
{ {
@@ -90,6 +82,7 @@ namespace StgSharp.Mathematics.Numeric
SleepMode afterWork SleepMode afterWork
) )
{ {
ObjectDisposedException.ThrowIf(_disposed, nameof(MatrixParallelWrap));
AfterWork = (ThreadPriority)afterWork; AfterWork = (ThreadPriority)afterWork;
foreach (MatrixParallelThread thread in _threads) foreach (MatrixParallelThread thread in _threads)
{ {
@@ -101,7 +94,9 @@ namespace StgSharp.Mathematics.Numeric
public void WaitAllWraps() public void WaitAllWraps()
{ {
ObjectDisposedException.ThrowIf(_disposed, nameof(MatrixParallelWrap)); if (_disposed) {
return;
}
_countdownEvent.Wait(); _countdownEvent.Wait();
} }
+1 -1
View File
@@ -172,7 +172,7 @@
<!-- source files --> <!-- source files -->
<!-- ════════════════════════════════════════════════════════ --> <!-- ════════════════════════════════════════════════════════ -->
<ItemGroup> <ItemGroup>
<ClCompile Include="sn-src\m.basic.x86.c" /> <ClCompile Include="sn-src\m.transpose.x86.c" />
<ClCompile Include="sn-src\dllmain.c" /> <ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\m.bufop.x86.c" /> <ClCompile Include="sn-src\m.bufop.x86.c" />
<ClCompile Include="sn-src\m.solve.x86.c" /> <ClCompile Include="sn-src\m.solve.x86.c" />
@@ -39,7 +39,7 @@
<ClCompile Include="sn-src\load_glfw.c"> <ClCompile Include="sn-src\load_glfw.c">
<Filter>源文件</Filter> <Filter>源文件</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="sn-src\m.basic.x86.c"> <ClCompile Include="sn-src\m.transpose.x86.c">
<Filter>源文件</Filter> <Filter>源文件</Filter>
</ClCompile> </ClCompile>
<ClCompile Include="sn-src\sk.fma.x86.c"> <ClCompile Include="sn-src\sk.fma.x86.c">
+1 -20
View File
@@ -141,7 +141,7 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local()
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0); const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0);
const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0); const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0);
const int is_zhaoxin = (memcmp(vendor, " Shanghai ", 12) == 0); const int is_zhaoxin = (strcmp(vendor, " Shanghai ") == 0);
const int is_via = is_centaur || is_zhaoxin; const int is_via = is_centaur || is_zhaoxin;
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx); get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
@@ -350,23 +350,4 @@ SN_API SIMDID SN_DECL sn_get_simd_level_global()
return global_simdid; return global_simdid;
} }
#else
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID simdid;
simdid.mask = 0;
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#endif #endif
@@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
@@ -6,7 +6,7 @@ using System.Threading.Tasks;
namespace StgSharp.UnserInterface.View namespace StgSharp.UnserInterface.View
{ {
internal class PresentorDescription internal class PresenterDescription
{ {
} }
} }
+2 -2
View File
@@ -38,7 +38,7 @@ namespace StgSharp
// 仅供 L4 内存安全测试使用。 // 仅供 L4 内存安全测试使用。
// ==================================================================== // ====================================================================
internal sealed unsafe class SequentialPredictor : IPrefetchPredict internal sealed unsafe class SequentialPredictor : IL4Predict
{ {
private readonly nuint[] _addresses; private readonly nuint[] _addresses;
@@ -313,7 +313,7 @@ namespace StgSharp
{ {
nuint[] result = new nuint[count]; nuint[] result = new nuint[count];
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
result[i] = (nuint)(0x1_0000_0000UL + (ulong)i * 0x1000); result[i] = (nuint)(0x1_0000_0000UL + (((ulong)i) * 0x1000));
} }
return result; return result;
} }
+10 -7
View File
@@ -40,7 +40,7 @@ namespace StgSharp
// WriteBack 记录被回写的 origin,供驱逐测试断言。 // WriteBack 记录被回写的 origin,供驱逐测试断言。
// ==================================================================== // ====================================================================
internal sealed unsafe class TrackingPredictor : IPrefetchPredict internal sealed unsafe class TrackingPredictor : IL4Predict
{ {
private readonly nuint[] _addresses; private readonly nuint[] _addresses;
@@ -128,7 +128,10 @@ namespace StgSharp
private static unsafe byte L4ReadFirstByte( private static unsafe byte L4ReadFirstByte(
L4.CacheLine line L4.CacheLine line
) => *(byte*)line.Address; )
{
return *(byte*)line.Address;
}
// ================================================================ // ================================================================
// P1-2 AgeAll_SaturatesAt3 // P1-2 AgeAll_SaturatesAt3
@@ -168,7 +171,7 @@ namespace StgSharp
for (int i = 0; i < snap.MaxId; i++) { for (int i = 0; i < snap.MaxId; i++) {
L4Assert( L4Assert(
snap.RrpvSnapshot[i] == 0, snap.RrpvSnapshot[i] == 0,
$"RrpvSnapshot[{i}] should be 0 right after Map, " + $"got {snap.RrpvSnapshot[i]}"); $"{$"RrpvSnapshot[{i}] should be 0 right after Map, "}{$"got {snap.RrpvSnapshot[i]}"}");
} }
} }
@@ -191,7 +194,7 @@ namespace StgSharp
if (snap.RrpvSnapshot[i] != 3) if (snap.RrpvSnapshot[i] != 3)
{ {
L4Assert(false, L4Assert(false,
$"RrpvSnapshot[{i}] == {snap.RrpvSnapshot[i]}, " + "expected 3 after 4x AgeAll (saturation failure or overflow)"); $"{$"RrpvSnapshot[{i}] == {snap.RrpvSnapshot[i]}, "}expected 3 after 4x AgeAll (saturation failure or overflow)");
allSaturated = false; allSaturated = false;
} }
} }
@@ -305,7 +308,7 @@ namespace StgSharp
} }
L4Assert( L4Assert(
evictedOrigin != nuint.Zero, evictedOrigin != nuint.Zero,
$"WriteBack origin should be a batch1 address, " + $"WriteBackLog=[{string.Join(", ", predictor2.WriteBackLog)}]"); $"{$"WriteBack origin should be a batch1 address, "}{($"WriteBackLog=[{string.Join(", ", predictor2.WriteBackLog)}]")}");
// 断言 c: SwissTable 中已不含该 origin // 断言 c: SwissTable 中已不含该 origin
L4Assert( L4Assert(
@@ -320,11 +323,11 @@ namespace StgSharp
remapped.Dispose(); remapped.Dispose();
L4Assert( L4Assert(
refill == 0xCC, refill == 0xCC,
$"Re-mapped evicted line should be re-prefetched with 0xCC, " + $"got 0x{refill:X2} (0xAA = stale data not evicted)"); $"{$"Re-mapped evicted line should be re-prefetched with 0xCC, "}{($"got 0x{refill:X2} (0xAA = stale data not evicted)")}");
if (_l4FailCount == 0) { if (_l4FailCount == 0) {
Console.WriteLine( Console.WriteLine(
$"PASS (evicted 0x{evictedOrigin:X}, remain={remain}, " + $"WriteBack×{predictor2.WriteBackLog.Count})"); $"{($"PASS (evicted 0x{evictedOrigin:X}, remain={remain}, ")}{$"WriteBack×{predictor2.WriteBackLog.Count})"}");
} }
} }
catch (Exception ex) catch (Exception ex)
+8 -8
View File
@@ -40,7 +40,7 @@ namespace StgSharp
// 用于 P2-2 验证"预填充后 Map 命中不触发二次 Prefetch"。 // 用于 P2-2 验证"预填充后 Map 命中不触发二次 Prefetch"。
// ==================================================================== // ====================================================================
internal sealed class CountingPredictor : IPrefetchPredict internal sealed class CountingPredictor : IL4Predict
{ {
private readonly nuint[] _addresses; private readonly nuint[] _addresses;
@@ -158,7 +158,7 @@ namespace StgSharp
L4.DebugSnapshot snap = l4.GetDebugSnapshot(); L4.DebugSnapshot snap = l4.GetDebugSnapshot();
Console.WriteLine(); Console.WriteLine();
Console.WriteLine( Console.WriteLine(
$" [diag] after warm-up: MaxId={snap.MaxId}, " + $"AheadCount={snap.AheadCount}, " + $"Predict×{predictAfterWarmup}, " + $"Prefetch×{prefetchAfterWarmup}"); $"{$" [diag] after warm-up: MaxId={snap.MaxId}, "}{$"AheadCount={snap.AheadCount}, "}{$"Predict×{predictAfterWarmup}, "}{$"Prefetch×{prefetchAfterWarmup}"}");
} }
// 断言 c(warm-up 后):所有 K 条地址均在缓存中 // 断言 c(warm-up 后):所有 K 条地址均在缓存中
@@ -195,7 +195,7 @@ namespace StgSharp
{ {
L4.DebugSnapshot snap = l4.GetDebugSnapshot(); L4.DebugSnapshot snap = l4.GetDebugSnapshot();
Console.WriteLine( Console.WriteLine(
$" [diag] after {K}x Map: AheadCount={snap.AheadCount}, " + $"Predict×{predictAfterMaps}, " + $"Prefetch×{prefetchAfterMaps}"); $"{$" [diag] after {K}x Map: AheadCount={snap.AheadCount}, "}{$"Predict×{predictAfterMaps}, "}{$"Prefetch×{prefetchAfterMaps}"}");
} }
// 断言 b: 已缓存的行不会被重新装填(Prefetch(line) 调用次数不增加) // 断言 b: 已缓存的行不会被重新装填(Prefetch(line) 调用次数不增加)
@@ -204,15 +204,15 @@ namespace StgSharp
// 这是预期行为,不断言 Predict 次数。 // 这是预期行为,不断言 Predict 次数。
L4Assert( L4Assert(
prefetchAfterMaps == prefetchAfterWarmup, prefetchAfterMaps == prefetchAfterWarmup,
$"Prefetch(line) should not be called again for already-cached addresses, " + $"was {prefetchAfterWarmup} after warm-up, now {prefetchAfterMaps}"); $"{$"Prefetch(line) should not be called again for already-cached addresses, "}{$"was {prefetchAfterWarmup} after warm-up, now {prefetchAfterMaps}"}");
// Predict 增加是正常的,仅打印供参考 // Predict 增加是正常的,仅打印供参考
Console.WriteLine( Console.WriteLine(
$" [info] Predict called {predictAfterMaps - predictAfterWarmup} " + $"extra time(s) during {K}x Map " + $"(aheadCount stays below threshold=1024 " + $"because K={K} << 1024, re-prediction is expected)"); $"{$" [info] Predict called {predictAfterMaps - predictAfterWarmup} "}{$"extra time(s) during {K}x Map "}{$"(aheadCount stays below threshold=1024 "}{$"because K={K} << 1024, re-prediction is expected)"}");
if (_l4FailCount == 0) { if (_l4FailCount == 0) {
Console.WriteLine( Console.WriteLine(
$"PASS (K={K}, all hits, no re-prefetch, " + $"Prefetch×{prefetchAfterMaps})"); $"{$"PASS (K={K}, all hits, no re-prefetch, "}{$"Prefetch×{prefetchAfterMaps})"}");
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -284,7 +284,7 @@ namespace StgSharp
// 此处仅记录行为,不判定 FAIL——但打印出来供后续决策参考。 // 此处仅记录行为,不判定 FAIL——但打印出来供后续决策参考。
Console.WriteLine( Console.WriteLine(
evictThrew ? evictThrew ?
"PASS (DebugEvict with null Predictor threw NullRef as expected " + "— caller is responsible for null check per trusted-call model)" : ("PASS (DebugEvict with null Predictor threw NullRef as expected " + "— caller is responsible for null check per trusted-call model)") :
"PASS (DebugEvict with null Predictor did not throw)"); "PASS (DebugEvict with null Predictor did not throw)");
// 核心断言:Prefetch() 本身在 null Predictor 时不崩溃 // 核心断言:Prefetch() 本身在 null Predictor 时不崩溃
@@ -300,7 +300,7 @@ namespace StgSharp
L4.DebugSnapshot snap = l4b.GetDebugSnapshot(); L4.DebugSnapshot snap = l4b.GetDebugSnapshot();
Console.WriteLine( Console.WriteLine(
$" [info] AheadCount after warm-up = {snap.AheadCount} " + $"(Prefetch triggers when < 1024)"); $"{$" [info] AheadCount after warm-up = {snap.AheadCount} "}{$"(Prefetch triggers when < 1024)"}");
// Prefetch 的 null 检查测试:直接调用 DebugPrefetch(如已暴露) // Prefetch 的 null 检查测试:直接调用 DebugPrefetch(如已暴露)
// 当前 Debug 接口未暴露 DebugPrefetch,改为间接验证: // 当前 Debug 接口未暴露 DebugPrefetch,改为间接验证: