From f7187ca7fde9c19decfab843b84045b3ce015209 Mon Sep 17 00:00:00 2001 From: Youmu Liu <105042599+Nitload-NSI@users.noreply.github.com> Date: Thu, 9 Apr 2026 18:07:49 +0800 Subject: [PATCH] add multi predictor support to l4, working for ver 5 of mat parallel --- StgSharp.Common/Collections/ConcurrentList.cs | 100 +++++++++++++ StgSharp.Common/Collections/ListSegment.cs | 39 +++-- StgSharp.Common/Collections/SwissTable.X86.cs | 2 - StgSharp.Common/Environment/Platform.cs | 7 +- .../HighPerformance/Memory/L4.CacheLine.cs | 47 +++--- .../HighPerformance/Memory/L4.Evict.cs | 25 +--- .../HighPerformance/Memory/L4.Exceptions.cs | 18 +-- .../Memory/L4.IntrinsicKernel.cs | 105 +++++++++++++ .../HighPerformance/Memory/L4.Predict.cs | 114 ++++++--------- .../HighPerformance/Memory/L4.Predictor.cs | 138 ++++++++++++++++++ .../HighPerformance/Memory/L4.Prefetch.cs | 7 +- StgSharp.Common/HighPerformance/Memory/L4.cs | 10 +- .../ProcessorAbstraction/NumaNode.cs | 1 - .../Numeric/Linear/MatrixMetadata.cs | 20 +-- .../Numeric/Runtime/MatrixEnumerator.cs | 97 ++++++++++++ .../Numeric/Runtime/MatrixOperatorPolicy.cs | 15 ++ .../Numeric/Runtime/MatrixParallelQueue.cs | 50 +++---- .../MatrixParallelThread.Sequential.cs | 12 ++ .../Numeric/Runtime/MatrixParallelThread.cs | 33 +++-- 19 files changed, 653 insertions(+), 187 deletions(-) create mode 100644 StgSharp.Common/Collections/ConcurrentList.cs rename StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.Predictor.cs => StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs (80%) create mode 100644 StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs create mode 100644 StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs create mode 100644 StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs create mode 100644 StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs create mode 100644 StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.Sequential.cs diff --git a/StgSharp.Common/Collections/ConcurrentList.cs b/StgSharp.Common/Collections/ConcurrentList.cs new file mode 100644 index 00000000..a37a6dda --- /dev/null +++ b/StgSharp.Common/Collections/ConcurrentList.cs @@ -0,0 +1,100 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="ConcurrentList" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Common.Collections +{ + public class ConcurrentFlexibleArray( + int capacity = 4 + ) + { + + private volatile T[] _array = new T[capacity]; + private readonly ReaderWriterLockSlim _rwLock = new(); + + public T this[ + int index + ] + { + get + { + _rwLock.EnterReadLock(); + try + { + if ((uint)index >= (uint)_array.Length) { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + return _array[index]; + } + finally + { + _rwLock.ExitReadLock(); + } + } + set + { + if ((uint)index >= (uint)_array.Length) + { + _rwLock.EnterWriteLock(); + try + { + if ((uint)index >= (uint)_array.Length) + { + int newSize = Math.Max(_array.Length * 2, index + 1); + T[] newArray = new T[newSize]; + Array.Copy(_array, newArray, _array.Length); + _array = newArray; + _array[index] = value; + } + } + finally + { + _rwLock.ExitWriteLock(); + } + } else + { + _rwLock.EnterReadLock(); + try + { + _array[index] = value; + } + finally + { + _rwLock.ExitReadLock(); + } + } + } + } + + } +} diff --git a/StgSharp.Common/Collections/ListSegment.cs b/StgSharp.Common/Collections/ListSegment.cs index 1185648d..c29488d0 100644 --- a/StgSharp.Common/Collections/ListSegment.cs +++ b/StgSharp.Common/Collections/ListSegment.cs @@ -1,4 +1,4 @@ -//----------------------------------------------------------------------- +//----------------------------------------------------------------------- // ----------------------------------------------------------------------- // file="ListSegment" // Project: StgSharp @@ -50,7 +50,11 @@ namespace StgSharp.Collections private int _length; private List _collection; - public ListSegment(List collection, int begin, int length) + public ListSegment( + List collection, + int begin, + int length + ) { ArgumentNullException.ThrowIfNull(collection); ArgumentOutOfRangeException.ThrowIfLessThan(begin, 0); @@ -61,7 +65,9 @@ namespace StgSharp.Collections _length = length; } - public T this[int indexValue] + public T this[ + int indexValue + ] { get { @@ -71,11 +77,13 @@ namespace StgSharp.Collections } } - public T this[Index i] + public T this[ + Index i + ] { get { - int offset = i.IsFromEnd ? _length - i.Value : i.Value; + int offset = i.IsFromEnd ? (_length - i.Value) : i.Value; ArgumentOutOfRangeException.ThrowIfLessThan(offset, 0); ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(offset, _length); return _collection[offset + _begin]; @@ -86,17 +94,22 @@ namespace StgSharp.Collections public int Count => _length; - public bool Contains(T item) + public bool Contains( + T item + ) { int index = _collection.IndexOf(item); - return index >= _begin && index < _length + index; + return (index >= _begin) && (index < _length + index); } - public void CopyTo(T[] array, int arrayIndex) + public void CopyTo( + T[] array, + int arrayIndex + ) { ArgumentNullException.ThrowIfNull(array); Span span = CollectionsMarshal.AsSpan(_collection).Slice(_begin, _length); - Span target = new Span(array, arrayIndex, _length); + Span target = new(array, arrayIndex, _length); span.CopyTo(target); } @@ -107,7 +120,9 @@ namespace StgSharp.Collections } } - void ICollection.Add(T item) + void ICollection.Add( + T item + ) { throw new NotSupportedException(); } @@ -117,7 +132,9 @@ namespace StgSharp.Collections throw new NotSupportedException(); } - bool ICollection.Remove(T item) + bool ICollection.Remove( + T item + ) { throw new NotSupportedException(); } diff --git a/StgSharp.Common/Collections/SwissTable.X86.cs b/StgSharp.Common/Collections/SwissTable.X86.cs index cf1da9b9..d6b8d1e0 100644 --- a/StgSharp.Common/Collections/SwissTable.X86.cs +++ b/StgSharp.Common/Collections/SwissTable.X86.cs @@ -107,8 +107,6 @@ namespace StgSharp.Collections byte ctrl = (byte)((hash >> EntryMaskWidth) & 0x7F); // H2: upper 8 bits (highest bit is naturally 0) Vector128 ctrlVector = Vector128.Create(ctrl); - int firstAvailableSlot = -1; // Record the first insertable slot - Vector128 bucketCtrl = Sse2.LoadVector128( (byte*)&bucket->Control); diff --git a/StgSharp.Common/Environment/Platform.cs b/StgSharp.Common/Environment/Platform.cs index e5298b6e..c8b71c9d 100644 --- a/StgSharp.Common/Environment/Platform.cs +++ b/StgSharp.Common/Environment/Platform.cs @@ -25,6 +25,7 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Common.HighPerformance.ProcessorAbstraction; using StgSharp.Mathematics; using System; using System.Reflection; @@ -211,6 +212,8 @@ namespace StgSharp if (Dialogue.NeedShowWhenStartup) { Dialogue.CreateDialogueProcessIfNotExist(); } + + Numa.Initialize(); /**/ } @@ -228,10 +231,10 @@ namespace StgSharp public void UninitializeModule() { - throw new NotImplementedException(); + return; } } } -} \ No newline at end of file +} diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs index ddfd9438..d9ed91e4 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -1,30 +1,27 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="L4.CacheLine.cs" +// file="L4.CacheLine" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the “Software”), to deal in the Software without restriction, -// including without limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of the Software, -// and to permit persons to whom the Software is furnished to do so, -// subject to the following conditions: -// -// The above copyright notice and -// this permission notice shall be included in all copies -// or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED “AS IS”, -// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH -// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -51,8 +48,8 @@ namespace StgSharp.HighPerformance.Memory internal int _refCount; internal nuint _address; internal nuint _origin; - - internal ulong _profile; + internal uint _predictor; + internal uint _profile; } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index d2fe4d67..463fb5c4 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -42,16 +42,7 @@ namespace StgSharp.HighPerformance.Memory [MethodImpl(MethodImplOptions.AggressiveInlining)] private unsafe void AgeAll() { - Vector128 one = Vector128.Create((byte)0x01); - Vector128 max = Vector128.Create((byte)0x03); - byte* p = (byte*)_eviction; - for (int i = 0; i < CacheLineCount; i += 16) - { - Vector128 v = Sse2.LoadVector128(p + i); - v = Sse2.AddSaturate(v, one); - v = Sse2.Min(v, max); - Sse2.Store(p + i, v); - } + _intrinsic.AgeAll(_eviction); } /// @@ -69,15 +60,12 @@ namespace StgSharp.HighPerformance.Memory ) { int remain = count; - Vector128 one = Vector128.Create((byte)0x01); Vector128 three = Vector128.Create((byte)0x03); byte* p = (byte*)_eviction; - for (int i = 0; i < CacheLineCount && remain > 0; i += 16) + for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16) { - Vector128 v = Sse2.LoadVector128(p + i); - Vector128 mask = Sse2.CompareEqual(v, three); - int evictMask = Sse2.MoveMask(mask); - while (evictMask != 0 && remain > 0) + int evictMask = _intrinsic.ScanEvict(p + i, three); + while ((evictMask != 0) && (remain > 0)) { int bitPos = BitOperations.TrailingZeroCount(evictMask); int index = bitPos + i; @@ -86,7 +74,10 @@ namespace StgSharp.HighPerformance.Memory { // EvictLine(i + index); _ = _map.TryRemove(head->_origin, out _); - PrefetchPredict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan((byte*)head->_address, Unsafe.SizeOf())); + int id = (int)head->_predictor; + IL4Predict predict = _predictions[id] ?? + throw new L4PredictUnregisteredException(id); + predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan((byte*)head->_address, Unsafe.SizeOf())); RecycleLine(index); remain--; } diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.Predictor.cs b/StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs similarity index 80% rename from StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.Predictor.cs rename to StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs index a96ef1d5..162f4ae9 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.Predictor.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Exceptions.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="MatrixParallelQueue.Predictor" +// file="L4.Exceptions" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -25,25 +25,21 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- -using StgSharp.HighPerformance.Memory; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; -namespace StgSharp.Mathematics.Numeric.Runtime +namespace StgSharp.HighPerformance.Memory { - internal interface IMatrixPredictor : IL4Predict - where TSelf : unmanaged, IMatrixPredictor + public sealed class L4PredictUnregisteredException : Exception { - static abstract PredictPolicy Policy { get; } - } + public L4PredictUnregisteredException( + int id + ) + : base($"L4 predictor with id {id} is null. Maybe it has been unregistered.") { } - internal enum PredictPolicy - { - Transparent = 0, - Sequential = 1, } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs b/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs new file mode 100644 index 00000000..0bef7e23 --- /dev/null +++ b/StgSharp.Common/HighPerformance/Memory/L4.IntrinsicKernel.cs @@ -0,0 +1,105 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="L4.IntrinsicKernel" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Numerics; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.HighPerformance.Memory +{ + public unsafe partial class L4 + { + + private interface IIntrinsicKernel + { + + static IIntrinsicKernel Instance { get; } = Create(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AgeAll( + EvictionArray* eviction + ); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int ScanEvict( + byte* vPtr, + Vector128 three + ); + + private static IIntrinsicKernel Create() + { + if (Sse42.IsSupported) + { + return new SseKernel(); + } else + { + throw new PlatformNotSupportedException("No supported intrinsic kernel found for current platform."); + } + } + + } + + private struct SseKernel : IIntrinsicKernel + { + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AgeAll( + EvictionArray* eviction + ) + { + Vector128 one = Vector128.Create((byte)0x01); + Vector128 max = Vector128.Create((byte)0x03); + byte* p = (byte*)eviction; + for (int i = 0; i < CacheLineCount; i += 16) + { + Vector128 v = Sse2.LoadVector128(p + i); + v = Sse2.AddSaturate(v, one); + v = Sse2.Min(v, max); + Sse2.Store(p + i, v); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int ScanEvict( + byte* vPtr, + Vector128 three + ) + { + Vector128 v = Sse2.LoadVector128(vPtr); + Vector128 mask = Sse2.CompareEqual(v, three); + return Sse2.MoveMask(mask); + } + + } + + } +} diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs index 071149ca..1ee33e68 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="L4.Predict" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,8 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.Common.Collections; +using StgSharp.Mathematics.Memory; using System; using System.Collections.Generic; using System.Linq; @@ -33,85 +35,65 @@ using System.Threading.Tasks; namespace StgSharp.HighPerformance.Memory { - [Flags] - public enum PredictResult + public partial class L4 { - Success = 1, - Exhausted = 2, + private ConcurrentBufferStack _idReuse = new(32); - } + private ConcurrentFlexibleArray _predictions = new(32); - public interface IL4Predict - { + private int _largestId = 0; - PredictResult Predict(Span node, out int count); + private Queue _predictionQueue = []; - /// - /// Materializes the cache line identified by into . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Destination cache line buffer. - /// - void Prefetch(nuint origin, ulong policy, Span cache); + private IL4Predict CurrentPredict { get; set; } - /// - /// Writes the cache line in back to . - /// - /// - /// Source address associated with the cache line. - /// - /// - /// Mapping policy associated with the cache line. - /// - /// - /// Source cache line buffer. - /// - void WriteBack(nuint origin, ulong policy, ReadOnlySpan cache); - - } - - public struct CacheLinePrediction : IEquatable - { - - public override readonly bool Equals(object? obj) + public int RegisterPredict( + IL4Predict predict + ) { - return (obj is CacheLinePrediction prediction) && - Address.Equals(prediction.Address) && - (MapPolicy == prediction.MapPolicy); + if (predict is null) { + return -1; + } + if (!_idReuse.TryPop(out int id)) { + id = _largestId++; + } + _predictions[id] = predict; + _predictionQueue.Enqueue(id); + return id; } - public readonly bool Equals(CacheLinePrediction other) + public IL4Predict? UnregisterPredict( + int id + ) { - return - (Address == other.Address) && (MapPolicy == other.MapPolicy); + if (id < 0) { + return null; + } + IL4Predict p = _predictions[id]; + if (p == null) { + return p; + } + _predictions[id] = null!; + + // TODO: scan and write back + + _idReuse.Push(id); + return p; } - public override readonly int GetHashCode() + private void UseNewPredict() { - return HashCode.Combine(Address, MapPolicy); + if (_predictionQueue.Count == 0) { + return; + } + if (_predictionQueue.TryDequeue(out int id)) + { + IL4Predict predict = _predictions[id] ?? + throw new L4PredictUnregisteredException(id); + CurrentPredict = predict; + } } - public static bool operator !=(CacheLinePrediction left, CacheLinePrediction right) - { - return !(left == right); - } - public static bool operator ==(CacheLinePrediction left, CacheLinePrediction right) - { - return left.Equals(right); - } - -#pragma warning disable CA1051 - public nuint Address; - public ulong MapPolicy; - -#pragma warning restore CA1051 } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs b/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs new file mode 100644 index 00000000..b9f46b9b --- /dev/null +++ b/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs @@ -0,0 +1,138 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="L4.Predictor" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.HighPerformance.Memory +{ + [Flags] + public enum PredictResult + { + + Success = 1, + Exhausted = 2, + + } + + public interface IL4Predict + { + + PredictResult Predict( + Span node, + out int count + ); + + /// + /// Materializes the cache line identified by into . + /// + /// + /// Source address associated with the cache line. + /// + /// + /// Mapping policy associated with the cache line. + /// + /// + /// Destination cache line buffer. + /// + void Prefetch( + nuint origin, + ulong policy, + Span cache + ); + + /// + /// Writes the cache line in back to . + /// + /// + /// Source address associated with the cache line. + /// + /// + /// Mapping policy associated with the cache line. + /// + /// + /// Source cache line buffer. + /// + void WriteBack( + nuint origin, + uint policy, + ReadOnlySpan cache + ); + + } + + public struct CacheLinePrediction : IEquatable + { + + public override readonly bool Equals( + object? obj + ) + { + return (obj is CacheLinePrediction prediction) && + Address.Equals(prediction.Address) && + (MapPolicy == prediction.MapPolicy); + } + + public readonly bool Equals( + CacheLinePrediction other + ) + { + return + (Address == other.Address) && (MapPolicy == other.MapPolicy); + } + + public override readonly int GetHashCode() + { + return HashCode.Combine(Address, MapPolicy); + } + + public static bool operator !=( + CacheLinePrediction left, + CacheLinePrediction right + ) + { + return !(left == right); + } + public static bool operator ==( + CacheLinePrediction left, + CacheLinePrediction right + ) + { + return left.Equals(right); + } + +#pragma warning disable CA1051 + public nuint Address; + public uint MapPolicy; + internal uint Predictor; +#pragma warning restore CA1051 + } +} diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs index 52204682..761bf934 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs @@ -36,14 +36,15 @@ namespace StgSharp.HighPerformance.Memory private partial void Prefetch() { - if (PrefetchPredict is null) { + if (CurrentPredict is null) { return; } if (_activeCount > MaxActivateCacheLineCount) { throw new OverflowException("Too many activated cache line in external reference."); } Span span = new(_prediction, PredictionCount); - if (!PrefetchPredict.Predict(span, out int actualCount) && (actualCount <= 0)) + PredictResult result = PrefetchPredict.Predict(span, out int actualCount); + if ((result == PredictResult.Exhausted) && (actualCount <= 0)) { _exhausted = true; return; @@ -87,7 +88,7 @@ namespace StgSharp.HighPerformance.Memory head->_origin = address; head->_profile = handle.MapPolicy; head->_refCount = 0; - head->_address = nuint & _data[index]; + head->_address = (nuint)(_data + index); // pre fetch them here PrefetchPredict.Prefetch(head->_origin, head->_profile, new Span((byte*)head->_address, Unsafe.SizeOf())); diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index 92bbd0b8..83b1756e 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -38,12 +38,14 @@ namespace StgSharp.HighPerformance.Memory private const int PredictionCount = 2048; private const int PredictionLeastAhead = 1024; + private static readonly IIntrinsicKernel _intrinsic = IIntrinsicKernel.Instance; + private readonly CacheLineData* _data; private readonly EvictionArray* _eviction; private readonly CacheLineHead* _head; private readonly CacheLinePrediction* _prediction; - private bool _disposed; + private bool _exhausted; private int _activeCount; private int _aheadCount; @@ -84,7 +86,7 @@ namespace StgSharp.HighPerformance.Memory ptr += stackSize; } - public IL4Predict PrefetchPredict { get; set; } + // public IL4Predict PrefetchPredict { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void ContinueOnce( @@ -121,7 +123,7 @@ namespace StgSharp.HighPerformance.Memory nuint origin ) { - if ((_aheadCount < PredictionLeastAhead) && !PrefetchPredict.IsExhausted) + if ((_aheadCount < PredictionLeastAhead) && !_exhausted) { if (Interlocked.CompareExchange(ref _prefetching, 1, 0) == 0) { @@ -193,7 +195,7 @@ namespace StgSharp.HighPerformance.Memory _activeCount = ref activeCount; } - [Obsolete("You should not create a CacheLineHandle outside a L4", true)] + [Obsolete("You should not create a CacheLineHandle instance outside a L4", true)] public L4CacheLine() { } public readonly nuint Address => _handle; diff --git a/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs b/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs index 50a49977..8b7343c6 100644 --- a/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs +++ b/StgSharp.Common/HighPerformance/ProcessorAbstraction/NumaNode.cs @@ -42,7 +42,6 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction internal NumaNode( uint numaId, int count, - Span logicalCoreIndex, Span logicalCoreIds ) diff --git a/StgSharp.Mathematics/Numeric/Linear/MatrixMetadata.cs b/StgSharp.Mathematics/Numeric/Linear/MatrixMetadata.cs index bcc27cf9..40c32b87 100644 --- a/StgSharp.Mathematics/Numeric/Linear/MatrixMetadata.cs +++ b/StgSharp.Mathematics/Numeric/Linear/MatrixMetadata.cs @@ -90,10 +90,10 @@ namespace StgSharp.Mathematics.Numeric int yKernel ) { - return xKernel >= 0 && - yKernel >= 0 && - xKernel < enumerator.KernelColumnLength && - yKernel < enumerator.KernelRowLength; + return (xKernel >= 0) && + (yKernel >= 0) && + (xKernel < enumerator.KernelColumnLength) && + (yKernel < enumerator.KernelRowLength); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -104,7 +104,7 @@ namespace StgSharp.Mathematics.Numeric ) { long order = GetTriangularOrder(enumerator); - return xKernel >= 0 && yKernel >= 0 && xKernel < order && yKernel < order; + return (xKernel >= 0) && (yKernel >= 0) && (xKernel < order) && (yKernel < order); } private sealed class DenseSquareKernelBoarder : MatrixKernelBoarder @@ -122,7 +122,7 @@ namespace StgSharp.Mathematics.Numeric long col = enumerator.KernelColumnLength; // Console.Write($"Attempt to get offset at ({x},{y}), offset is {x * col + y} "); - return x * col + y; + return (x * col) + y; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -151,7 +151,7 @@ namespace StgSharp.Mathematics.Numeric long x = xKernel; long y = yKernel; - long prefixColumns = x * order - (x * (x - 1L)) / 2L; + long prefixColumns = (x * order) - ((x * (x - 1L)) / 2L); return prefixColumns + (y - x); } @@ -162,7 +162,8 @@ namespace StgSharp.Mathematics.Numeric in MatrixSize enumerator ) { - return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && yKernel >= xKernel; + return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && + (yKernel >= xKernel); } } @@ -191,7 +192,8 @@ namespace StgSharp.Mathematics.Numeric in MatrixSize enumerator ) { - return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && yKernel <= xKernel; + return IsWithinTriangularDomain(enumerator, xKernel, yKernel) && + (yKernel <= xKernel); } } diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs new file mode 100644 index 00000000..c1a277fd --- /dev/null +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixEnumerator.cs @@ -0,0 +1,97 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixEnumerator" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using StgSharp.HighPerformance.Memory; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric.Runtime +{ + public enum PredictPolicy + { + + Transparent = 0, + BufferSequential = 1, + SquareRowMajor = 2, + SquareColMajor = 3, + + } + + public unsafe interface IMatrixCachePredictor : IL4Predict + where T : unmanaged, INumber + where TSelf : class, IMatrixCachePredictor + { + + public bool NeedWriteBack { get; } + + public static abstract PredictPolicy Policy { get; set; } + + } + + public sealed unsafe class BufferSequentialPredictor : IMatrixCachePredictor, T> + where T : unmanaged, INumber + { + + private MatrixKernelEnumerator enumerator; + + public bool NeedWriteBack => throw new NotImplementedException(); + + public static PredictPolicy Policy { get; set; } = PredictPolicy.SquareRowMajor; + + private MatrixKernel* Buffer { get; set; } + + public PredictResult Predict( + Span node, + out int count + ) + { + throw new NotImplementedException(); + } + + public void Prefetch( + nuint origin, + ulong policy, + Span cache + ) + { + throw new NotImplementedException(); + } + + public void WriteBack( + nuint origin, + ulong policy, + ReadOnlySpan cache + ) + { + throw new NotImplementedException(); + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs new file mode 100644 index 00000000..2c04c556 --- /dev/null +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixOperatorPolicy.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric.Runtime +{ + internal enum MatrixOperatorPolicy + { + + Sequential = 1, + + } +} diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs index eea9cfce..eb0e1ebd 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelQueue.cs @@ -1,30 +1,27 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="MatrixParallelQueue.cs" +// file="MatrixParallelQueue" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // -// Permission is hereby granted, free of charge, to any person -// obtaining a copy of this software and associated documentation -// files (the “Software”), to deal in the Software without restriction, -// including without limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of the Software, -// and to permit persons to whom the Software is furnished to do so, -// subject to the following conditions: -// -// The above copyright notice and -// this permission notice shall be included in all copies -// or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED “AS IS”, -// WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -// ARISING FROM, OUT OF OR IN CONNECTION WITH -// THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- @@ -42,11 +39,14 @@ using System.Threading.Tasks; namespace StgSharp.Mathematics.Numeric { - internal unsafe ref struct MatrixParallelQueue where TPredictor: unmanaged, IMatrixPredictor + internal unsafe class MatrixParallelQueue where T : unmanaged, INumber + where TPredictor : class, IMatrixCachePredictor { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Steal(MatrixParallelTask* task) + public bool Steal( + MatrixParallelTask* task + ) { // TODO 使用提取的内容填充task throw new NotImplementedException(); diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.Sequential.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.Sequential.cs new file mode 100644 index 00000000..029007d8 --- /dev/null +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.Sequential.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric.Runtime +{ + internal class MatrixParallelThread + { + } +} diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs index 3a0de8f9..3fd64c58 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs @@ -2,8 +2,8 @@ // ----------------------------------------------------------------------- // file="MatrixParallelThread" // Project: StgSharp -// AuthorGroup: Nitload Space -// Copyright (c) Nitload Space. All rights reserved. +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -55,7 +55,9 @@ namespace StgSharp.Mathematics.Numeric private readonly ManualResetEventSlim _resetEvent; private Thread _managedThread; - internal MatrixParallelThread(int poolId) + internal MatrixParallelThread( + int poolId + ) { _resetEvent = new ManualResetEventSlim(false); _managedThread = new Thread(MatrixParallelWorkLoad); @@ -103,14 +105,18 @@ namespace StgSharp.Mathematics.Numeric } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetSingleThreadTask(ref MatrixParallelTask task) + internal void SetSingleThreadTask( + ref MatrixParallelTask task + ) { if (IsSingle) { *_currentTask = task; } } - protected virtual void Dispose(bool disposing) + protected virtual void Dispose( + bool disposing + ) { if (!_disposed) { @@ -172,7 +178,7 @@ namespace StgSharp.Mathematics.Numeric { case PredictPolicy.Transparent: break; - case PredictPolicy.Sequential: + case PredictPolicy.BufferSequential: break; default: break; @@ -202,19 +208,24 @@ namespace StgSharp.Mathematics.Numeric #pragma warning disable CA5393 [LibraryImport(Native.LibName, EntryPoint = "sn_get_simd_level_local")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] internal static partial ulong GetSimdId(); [LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] - internal static partial void SetThreadAffinity(int coreId); + internal static partial void SetThreadAffinity( + int coreId + ); [LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] - private static unsafe partial void LoadIntrinsicFunction(IntrinsicContext* context, ulong id); + private static unsafe partial void LoadIntrinsicFunction( + IntrinsicContext* context, + ulong id + ); #pragma warning restore CA5393 } }