diff --git a/StgSharp.Common/Collections/ConcurrentList.cs b/StgSharp.Common/Collections/ConcurrentFlexibaleArray.cs similarity index 87% rename from StgSharp.Common/Collections/ConcurrentList.cs rename to StgSharp.Common/Collections/ConcurrentFlexibaleArray.cs index a37a6dda..f823a888 100644 --- a/StgSharp.Common/Collections/ConcurrentList.cs +++ b/StgSharp.Common/Collections/ConcurrentFlexibaleArray.cs @@ -1,6 +1,6 @@ //----------------------------------------------------------------------- // ----------------------------------------------------------------------- -// file="ConcurrentList" +// file="ConcurrentFlexibaleArray" // Project: StgSharp // AuthorGroup: Nitload // Copyright (c) Nitload. All rights reserved. @@ -96,5 +96,23 @@ namespace StgSharp.Common.Collections } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Span AsSpan() + { + return _array.AsSpan(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ReleaseSpanOperation() + { + _rwLock.ExitReadLock(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void RequestSpanOperation() + { + _rwLock.EnterReadLock(); + } + } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs index d9ed91e4..363dc9fe 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.CacheLine.cs @@ -45,10 +45,10 @@ namespace StgSharp.HighPerformance.Memory { internal int _id; + internal int _predictor; internal int _refCount; internal nuint _address; internal nuint _origin; - internal uint _predictor; internal uint _profile; } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs index 463fb5c4..67ec04d8 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Evict.cs @@ -62,29 +62,38 @@ namespace StgSharp.HighPerformance.Memory int remain = count; Vector128 three = Vector128.Create((byte)0x03); byte* p = (byte*)_eviction; - for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16) + _predictors.RequestSpanOperation(); + Span pSpan = _predictors.AsSpan(); + try { - int evictMask = _intrinsic.ScanEvict(p + i, three); - while ((evictMask != 0) && (remain > 0)) + for (int i = 0; (i < CacheLineCount) && (remain > 0); i += 16) { - int bitPos = BitOperations.TrailingZeroCount(evictMask); - int index = bitPos + i; - CacheLineHead* head = &_head[index]; - if (Interlocked.CompareExchange(ref head->_refCount, int.MinValue, 0) == 0) + int evictMask = _intrinsic.ScanEvict(p + i, three); + while ((evictMask != 0) && (remain > 0)) { - // EvictLine(i + index); - _ = _map.TryRemove(head->_origin, out _); - int id = (int)head->_predictor; - IL4Predict predict = _predictions[id] ?? - throw new L4PredictUnregisteredException(id); - predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan((byte*)head->_address, Unsafe.SizeOf())); - RecycleLine(index); - remain--; + int bitPos = BitOperations.TrailingZeroCount(evictMask); + int index = bitPos + i; + CacheLineHead* head = &_head[index]; + if (Interlocked.CompareExchange(ref head->_refCount, int.MinValue, 0) == 0) + { + // EvictLine(i + index); + _ = _map.TryRemove(head->_origin, out _); + int id = (int)head->_predictor; + IL4Predict predict = pSpan[id] ?? + throw new L4PredictUnregisteredException(id); + predict.WriteBack(head->_origin, head->_profile, new ReadOnlySpan((byte*)head->_address, Unsafe.SizeOf())); + RecycleLine(index); + remain--; + } + evictMask &= evictMask - 1; } - evictMask &= evictMask - 1; } + return remain; + } + finally + { + _predictors.ReleaseSpanOperation(); } - return remain; } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs index 1ee33e68..f233b524 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Predict.cs @@ -28,6 +28,7 @@ using StgSharp.Common.Collections; using StgSharp.Mathematics.Memory; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Text; @@ -40,14 +41,16 @@ namespace StgSharp.HighPerformance.Memory private ConcurrentBufferStack _idReuse = new(32); - private ConcurrentFlexibleArray _predictions = new(32); + private ConcurrentFlexibleArray _predictors = new(32); - private int _largestId = 0; + private ConcurrentQueue _predictionQueue = new(); - private Queue _predictionQueue = []; + private int _largestId ; private IL4Predict CurrentPredict { get; set; } + private int CurrentPredictIndex { get; set; } + public int RegisterPredict( IL4Predict predict ) @@ -58,7 +61,7 @@ namespace StgSharp.HighPerformance.Memory if (!_idReuse.TryPop(out int id)) { id = _largestId++; } - _predictions[id] = predict; + _predictors[id] = predict; _predictionQueue.Enqueue(id); return id; } @@ -70,29 +73,30 @@ namespace StgSharp.HighPerformance.Memory if (id < 0) { return null; } - IL4Predict p = _predictions[id]; + IL4Predict p = _predictors[id]; if (p == null) { return p; } - _predictions[id] = null!; - - // TODO: scan and write back - + _predictors[id] = null!; _idReuse.Push(id); return p; } - private void UseNewPredict() + private bool TryGetNextPredict( + ) { - if (_predictionQueue.Count == 0) { - return; + if (_predictionQueue.IsEmpty) { + return false; } if (_predictionQueue.TryDequeue(out int id)) { - IL4Predict predict = _predictions[id] ?? + IL4Predict predict = _predictors[id] ?? throw new L4PredictUnregisteredException(id); CurrentPredict = predict; + CurrentPredictIndex = id; + return true; } + return false; } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs b/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs index b9f46b9b..b4faa043 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Predictor.cs @@ -45,6 +45,11 @@ namespace StgSharp.HighPerformance.Memory public interface IL4Predict { + bool IsSamePolicy( + uint policy, + IL4Predict predict + ); + PredictResult Predict( Span node, out int count @@ -65,7 +70,7 @@ namespace StgSharp.HighPerformance.Memory /// void Prefetch( nuint origin, - ulong policy, + uint policy, Span cache ); diff --git a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs index 761bf934..44f4c183 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.Prefetch.cs @@ -42,9 +42,13 @@ namespace StgSharp.HighPerformance.Memory if (_activeCount > MaxActivateCacheLineCount) { throw new OverflowException("Too many activated cache line in external reference."); } + IL4Predict predict = CurrentPredict; Span span = new(_prediction, PredictionCount); - PredictResult result = PrefetchPredict.Predict(span, out int actualCount); - if ((result == PredictResult.Exhausted) && (actualCount <= 0)) + PredictResult result = predict.Predict(span, out int actualCount); + if ((result & PredictResult.Exhausted) != 0) { + _exhausted = !TryGetNextPredict(); + } + if (actualCount <= 0) { _exhausted = true; return; @@ -62,53 +66,72 @@ namespace StgSharp.HighPerformance.Memory missIndex++; } } - - // now prefetch the misses - for (int i = 0; i < missIndex; i++) + _predictors.RequestSpanOperation(); + Span predictorSpan = _predictors.AsSpan(); + try { - ref CacheLinePrediction handle = ref span[i]; - nuint address = handle.Address; - if (TestHit(address)) + // now prefetch the misses + for (int i = 0; i < missIndex; i++) { - continue; - } + CacheLineHead* head; + ref CacheLinePrediction pHandle = ref span[i]; + nuint address = pHandle.Address; + int index; + if (_map.TryGet(address, out nuint handle)) + { + head = (CacheLineHead*)handle; + index = head->_id; + (*_eviction)[index] = 0; // reset eviction counter - int retryCount = 0, index, remain = missIndex - i; - const int maxRetry = 1024; - while (!TryGetEmptyLine(out index)) - { - remain = Evict(remain); - retryCount++; - if (retryCount > maxRetry) { - throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); + #region update predictor belonging + + int predictId = head->_predictor; + IL4Predict belong = predictorSpan[predictId]; + if (predict.IsSamePolicy(head->_profile, belong)) + { + head->_predictor = predictId; // wrong line + } else + { + belong.WriteBack(head->_origin, head->_profile, new ReadOnlySpan((byte*)head->_address, Unsafe.SizeOf())); + head->_profile = pHandle.MapPolicy; + predict.Prefetch(head->_origin, head->_profile, new Span((byte*)head->_address, Unsafe.SizeOf())); + } + + #endregion + + continue; } + + int retryCount = 0, remain = missIndex - i; + const int maxRetry = 1024; + while (!TryGetEmptyLine(out index)) + { + remain = Evict(remain); + retryCount++; + if (retryCount > maxRetry) { + throw new OverflowException($"Failed to evict cache line after {maxRetry} retries."); + } + if (retryCount > maxRetry / 2) { + Thread.Sleep(1); + } + } + head = &_head[index]; + head->_id = index; + head->_origin = address; + head->_profile = pHandle.MapPolicy; + head->_refCount = 0; + head->_address = (nuint)(_data + index); + + // pre fetch them here + predict.Prefetch(head->_origin, head->_profile, new Span((byte*)head->_address, Unsafe.SizeOf())); + _ = _map.TryAddOrSet(address, (nuint)head, out _); } - CacheLineHead* head = &_head[index]; - head->_id = index; - head->_origin = address; - head->_profile = handle.MapPolicy; - head->_refCount = 0; - head->_address = (nuint)(_data + index); - - // pre fetch them here - PrefetchPredict.Prefetch(head->_origin, head->_profile, new Span((byte*)head->_address, Unsafe.SizeOf())); - _ = _map.TryAddOrSet(address, (nuint)head, out _); + _ = Interlocked.Add(ref _aheadCount, actualCount); } - _ = Interlocked.Add(ref _aheadCount, actualCount); - } - - private partial bool TestHit( - nuint origin - ) - { - if (_map.TryGet(origin, out nuint handle)) + finally { - CacheLineHead* head = (CacheLineHead*)handle; - int index = head->_id; - (*_eviction)[index] = 0; // reset eviction counter - return true; + _predictors.ReleaseSpanOperation(); } - return false; } } diff --git a/StgSharp.Common/HighPerformance/Memory/L4.cs b/StgSharp.Common/HighPerformance/Memory/L4.cs index 83b1756e..279cbb23 100644 --- a/StgSharp.Common/HighPerformance/Memory/L4.cs +++ b/StgSharp.Common/HighPerformance/Memory/L4.cs @@ -26,6 +26,7 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using StgSharp.Collections; +using StgSharp.Common.HighPerformance.ProcessorAbstraction; using System.Data; using System.Runtime.InteropServices; @@ -49,14 +50,20 @@ namespace StgSharp.HighPerformance.Memory private int _activeCount; private int _aheadCount; + + private readonly int _bindedNuma; private int _maxId; private volatile int _prefetching; private readonly nuint _baseAddress; private readonly SwissTable _map; private readonly UnmanagedStack _idMux; - public L4() + public L4( + int bindedNuma + ) { + _bindedNuma = bindedNuma; + // init order is fallow by align requirement of each component. nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align @@ -67,7 +74,7 @@ namespace StgSharp.HighPerformance.Memory // malloc all memory at once, and free at once in Dispose. // This is to avoid fragmentation and improve performance. - _baseAddress = (nuint)NativeMemory.AlignedAlloc(cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize, 16); + _baseAddress = (nuint)Numa.Alloc(bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize); nuint ptr = _baseAddress; // init all components with the buffer, and move the buffer pointer forward. @@ -86,6 +93,8 @@ namespace StgSharp.HighPerformance.Memory ptr += stackSize; } + public static nuint AllocSize { get; } + // public IL4Predict PrefetchPredict { get; set; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -114,7 +123,15 @@ namespace StgSharp.HighPerformance.Memory _disposed = true; _map.Dispose(); _idMux.Dispose(); - NativeMemory.AlignedFree((void*)_baseAddress); + + + nuint cacheSize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // SIMDID, no less than 16 byte bit + nuint entrySize = ((nuint)Unsafe.SizeOf()) * CacheLineCount; // 16 byte align + nuint predictionSize = ((nuint)Unsafe.SizeOf()) * PredictionCount; //16 byte align + nuint mapSize = (nuint)SwissTable.RequestedNativeMemorySize; // 16 byte align + nuint rrpvSize = (nuint)Unsafe.SizeOf(); // 16 byte align + nuint stackSize = ((nuint)sizeof(int)) * CacheLineCount; // int align + Numa.Free((void*)_baseAddress, _bindedNuma, cacheSize + entrySize + stackSize + mapSize + predictionSize + rrpvSize); GC.SuppressFinalize(this); } @@ -164,10 +181,6 @@ namespace StgSharp.HighPerformance.Memory private partial void Prefetch(); - private partial bool TestHit( - nuint origin - ); - private struct CacheLineData { diff --git a/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs b/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs index b3bf4896..d4e6be0d 100644 --- a/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs +++ b/StgSharp.Common/HighPerformance/ProcessorAbstraction/Numa.cs @@ -25,8 +25,10 @@ // // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- +using StgSharp.HighPerformance.Memory; using System; using System.Collections.Generic; +using System.Diagnostics.Contracts; using System.Linq; using System.Runtime.InteropServices; using System.Text; @@ -43,6 +45,35 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction public static ReadOnlySpan AllNumaNode => _numaNodes; + public static unsafe void* Alloc( + int numaId, + nuint size + ) + { + if (_numaNodes.Length == 0) { + throw new InvalidOperationException("Numa info has not been loaded."); + } + if ((numaId < 0) || (numaId >= _numaNodes.Length)) { + throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID."); + } + return NumaAllocNative(size, numaId); + } + + public static unsafe void Free( + void* address, + int numaId, + nuint size + ) + { + if (_numaNodes.Length == 0) { + throw new InvalidOperationException("Numa info has not been loaded."); + } + if ((numaId < 0) || (numaId >= _numaNodes.Length)) { + throw new ArgumentOutOfRangeException(nameof(numaId), "Invalid NUMA node ID."); + } + NumaFreeNative(address, size, numaId); + } + [LibraryImport(Native.LibName, EntryPoint = "sn_get_numa_each")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] @@ -60,6 +91,20 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction Span coreCountInNuma ); + public static L4[] RequestBufferForEachNuma() + { + if (_numaNodes.Length == 0) { + throw new InvalidOperationException("Numa info has not been loaded."); + } + L4[] buffers = new L4[NumaNodeCount]; + for (int i = 0; i < NumaNodeCount; i++) + { + L4 buffer = new L4(i); + buffers[i] = buffer; + } + return buffers; + } + internal static void Initialize() { Span coreCountInNuma = stackalloc int[(Environment.ProcessorCount / 32) + 1]; @@ -73,6 +118,13 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction } } + [LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] + internal static partial void SetThreadAffinity( + int coreId + ); + [MethodImpl(MethodImplOptions.NoInlining)] private static NumaNode InitNumaNode( uint numaId, @@ -84,5 +136,22 @@ namespace StgSharp.Common.HighPerformance.ProcessorAbstraction return new(numaId, count, globalIndex); } + [LibraryImport(Native.LibName, EntryPoint = "sn_numa_alloc")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] + private static unsafe partial void* NumaAllocNative( + nuint size, + int numaId + ); + + [LibraryImport(Native.LibName, EntryPoint = "sn_numa_free")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] + private static unsafe partial void NumaFreeNative( + void* ptr, + nuint size, + int numa_id + ); + } } diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Await.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Await.old similarity index 100% rename from StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Await.cs rename to StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Await.old diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.AwaitingQueue.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.AwaitingQueue.old similarity index 100% rename from StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.AwaitingQueue.cs rename to StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.AwaitingQueue.old diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Numa.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Numa.cs new file mode 100644 index 00000000..73cc78fe --- /dev/null +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallel.Numa.cs @@ -0,0 +1,48 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="MatrixParallel.Numa" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace StgSharp.Mathematics.Numeric +{ + public static unsafe partial class MatrixParallel + { + + private static int[] NumaPayload; + private static MatrixParallelThread[,] NumaThreads; + + private static void UpdateThreadPayload() + { + for (int i = 0; i < NumaPayload.Length; i++) { } + } + + } +} diff --git a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs index 3fd64c58..08834cb1 100644 --- a/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs +++ b/StgSharp.Mathematics/Numeric/Runtime/MatrixParallelThread.cs @@ -69,12 +69,8 @@ namespace StgSharp.Mathematics.Numeric public ThreadPriority AfterWork { get; set; } - internal ref readonly IntrinsicContext Intrinsic => ref (*_context); - internal bool IsSingle { get; set; } - internal MatrixParallelWrap BindedWrap { get; set; } - internal MatrixThreadState State { get; private set; } internal SIMDID Simd { get; private set; } @@ -163,7 +159,6 @@ namespace StgSharp.Mathematics.Numeric } State = MatrixThreadState.Idle; State = MatrixThreadState.Running; - AfterWork = BindedWrap.AfterWork; _managedThread.Priority = ThreadPriority.Highest; if (IsSingle) { @@ -173,17 +168,6 @@ namespace StgSharp.Mathematics.Numeric IsSingle = false; } else { - PredictPolicy policy = BindedWrap.Policy; - switch (policy) - { - case PredictPolicy.Transparent: - break; - case PredictPolicy.BufferSequential: - break; - default: - break; - } - // MatrixParallelQueue taskQueue = BindedWrap.TaskQueue; /* if (0 > (int)p ->ComputeHandle) { @@ -198,7 +182,6 @@ namespace StgSharp.Mathematics.Numeric break; } /**/ - BindedWrap.ReportThreadAccomplished(); } _managedThread.Priority = AfterWork; State = MatrixThreadState.Idle; @@ -208,19 +191,19 @@ namespace StgSharp.Mathematics.Numeric #pragma warning disable CA5393 [LibraryImport(Native.LibName, EntryPoint = "sn_get_simd_level_local")] - [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] internal static partial ulong GetSimdId(); [LibraryImport(Native.LibName, EntryPoint = "sn_set_thread_affinity")] - [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] internal static partial void SetThreadAffinity( int coreId ); [LibraryImport(Native.LibName, EntryPoint = "load_intrinsic_function")] - [UnmanagedCallConv(CallConvs = [ typeof(CallConvCdecl) ])] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] [DefaultDllImportSearchPaths(DllImportSearchPath.AssemblyDirectory)] private static unsafe partial void LoadIntrinsicFunction( IntrinsicContext* context, diff --git a/StgSharp.Native/sn-head/StgSharpC.h b/StgSharp.Native/sn-head/StgSharpC.h deleted file mode 100644 index 7730729e..00000000 --- a/StgSharp.Native/sn-head/StgSharpC.h +++ /dev/null @@ -1,73 +0,0 @@ -#ifdef _MSC_VER -#pragma once -#endif - -#ifndef SSC -#define SSC - -#include "ssc_internal.h" - -#define STBIDEF SSCAPI -#include "stb_image.h" - -#include "gl.h" -#include "wgl.h" -#include "glfw3.h" -#include -#include - -#ifndef SSCAPI_DEFINE - -#ifdef _WIN32 -#define SSCAPI __declspec(dllexport) -#elif defined _LINUX_ -#define SSCAPI __attribute__((dllexport)) -#elif defined _APPLE_ -#define SSCAPI __attribute__((visibility("default"))) -#else -#define SSCAPI -#endif - -#endif // ! SSCAPI_DEFINE - -#if __cplusplus -extern "C" { -#endif - -#define byte char -#define cs_char uint16_t -#define ssc_null_assert(ptr) assert(ptr != NULL) - -#define ALIGN(simdVec) _mm_loadu_ps(&simdVec) -#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec) -#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec) - -typedef struct Image { - int width; - int height; - int channel; - char *pixelPtr; -} Image; - -typedef char *(*imageLoader)(char const *filename, int *x, int *y, int *channels_in_file, - int desired_channels); - -SSCAPI char infolog[512]; -SSCAPI int SSCDECL glCheckShaderStat(GladGLContext *context, uint64_t shaderHandle, int key, - char **logRef); -SSCAPI void SSCDECL initGL(int majorVersion, int minorVersion); -SSCAPI unsigned int SSCDECL linkShaderProgram(GladGLContext *context, GLuint shadertype); -SSCAPI void SSCDECL loadImageData(char *location, Image *out, imageLoader loader); -SSCAPI GLFWglproc SSCDECL loadGlfuncDefault(char *procName); -SSCAPI void SSCDECL unloadImageData(Image *out); -SSCAPI char *SSCDECL readLog(void); - -#pragma endregion - -SSCAPI int SSCDECL load_intrinsic_function(void *intrinsic_context); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/StgSharp.Native/sn-head/StgSharpNative.h b/StgSharp.Native/sn-head/StgSharpNative.h index c2b98863..94fd5fe9 100644 --- a/StgSharp.Native/sn-head/StgSharpNative.h +++ b/StgSharp.Native/sn-head/StgSharpNative.h @@ -55,7 +55,8 @@ SN_API SIMDID SN_DECL sn_get_simd_level_local(); SN_API SIMDID SN_DECL sn_get_unite_simd(SIMDID left, SIMDID right); SN_API int SN_DECL sn_compare_simd(SIMDID left, SIMDID right); SN_API void SN_DECL sn_set_thread_affinity(int core_id); - +SN_API void* SN_DECL sn_numa_alloc(size_t size, int numa_id); +SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id); SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global); SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa); #endif diff --git a/StgSharp.Native/sn-src/thread.linux.c b/StgSharp.Native/sn-src/thread.linux.c index e0eb9f68..235b88f5 100644 --- a/StgSharp.Native/sn-src/thread.linux.c +++ b/StgSharp.Native/sn-src/thread.linux.c @@ -1,25 +1,236 @@ +#define _GNU_SOURCE + #include "sn_target.h" -#include "StgSharpNative.h" #if SN_IS_OS(SN_OS_LINUX) -#define _GNU_SOURCE +#include +#include +#include +#include #include #include +#include +#include +#include +#include +#include #include +typedef int (*sn_linux_id_visitor)(int value, void *context); + +typedef struct sn_linux_counter_context { + int count; +} sn_linux_counter_context; + +typedef struct sn_linux_max_context { + int max_value; +} sn_linux_max_context; + +typedef struct sn_linux_fill_cpu_context { + uint64_t *internal; + uint32_t *global; + int index; +} sn_linux_fill_cpu_context; + +typedef struct sn_linux_fill_node_count_context { + int *count_of_each_numa; +} sn_linux_fill_node_count_context; + +static int sn_linux_read_first_line(char const *path, char *buffer, size_t buffer_size) +{ + FILE *file = fopen(path, "r"); + if (file == NULL) + return 0; + + char *line = fgets(buffer, (int)buffer_size, file); + fclose(file); + return line != NULL; +} + +static int sn_linux_parse_nonnegative_int(char const **cursor, int *value) +{ + char *end = NULL; + uintmax_t parsed_value = 0; + + if (**cursor == '-') + return 0; + + errno = 0; + parsed_value = strtoumax(*cursor, &end, 10); + if (end == *cursor || errno == ERANGE || parsed_value > (uintmax_t)INT_MAX) + return 0; + + *cursor = end; + *value = (int)parsed_value; + return 1; +} + +static int sn_linux_visit_id_list(char const *text, sn_linux_id_visitor visitor, void *context) +{ + char const *cursor = text; + + while (*cursor != '\0') { + while (*cursor == ' ' || *cursor == '\t' || *cursor == '\n' || *cursor == ',') + cursor++; + + if (*cursor == '\0') + break; + + int begin = 0; + if (!sn_linux_parse_nonnegative_int(&cursor, &begin)) + return 0; + + int finish = begin; + + if (*cursor == '-') { + cursor++; + if (!sn_linux_parse_nonnegative_int(&cursor, &finish)) + return 0; + } + + if (finish < begin) + return 0; + + for (int value = begin;; value++) { + if (!visitor(value, context)) + return 0; + + if (value == finish) + break; + } + } + + return 1; +} + +static int sn_linux_count_id(int value, void *context) +{ + (void)value; + ((sn_linux_counter_context *)context)->count++; + return 1; +} + +static int sn_linux_track_max_id(int value, void *context) +{ + sn_linux_max_context *max_context = (sn_linux_max_context *)context; + if (value > max_context->max_value) + max_context->max_value = value; + return 1; +} + +static int sn_linux_fill_cpu_id(int value, void *context) +{ + sn_linux_fill_cpu_context *fill_context = (sn_linux_fill_cpu_context *)context; + + if (fill_context->internal != NULL) + fill_context->internal[fill_context->index] = (uint64_t)fill_context->index; + if (fill_context->global != NULL) + fill_context->global[fill_context->index] = (uint32_t)value; + + fill_context->index++; + return 1; +} + +static int sn_linux_count_cpu_list(char const *path) +{ + char buffer[256]; + if (!sn_linux_read_first_line(path, buffer, sizeof(buffer))) + return 0; + + sn_linux_counter_context counter = {0}; + if (!sn_linux_visit_id_list(buffer, sn_linux_count_id, &counter)) + return 0; + + return counter.count; +} + +static int sn_linux_fill_node_cpu_count(int value, void *context) +{ + char path[128]; + snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value); + + ((sn_linux_fill_node_count_context *)context)->count_of_each_numa[value] = + sn_linux_count_cpu_list(path); + return 1; +} + +static int sn_linux_node_exists(int numa_id) +{ + char path[128]; + snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id); + return access(path, F_OK) == 0; +} + +static size_t sn_linux_round_up_to_page_size(size_t size) +{ + int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE); + size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u; + + size_t rounded = size + effective_page_size - 1; + if (rounded < size) + return 0; + + return rounded - (rounded % effective_page_size); +} + +static int sn_linux_bind_memory_to_node(void *address, size_t size, int numa_id, + uint32_t flags) +{ +#if defined(SYS_mbind) + size_t bits_per_word = sizeof(unsigned long) * 8u; + size_t word_count = ((size_t)numa_id / bits_per_word) + 1u; + unsigned long *node_mask = (unsigned long *)calloc(word_count, sizeof(unsigned long)); + if (node_mask == NULL) + return 0; + + node_mask[(size_t)numa_id / bits_per_word] = 1ul << ((size_t)numa_id % bits_per_word); + + int syscall_result = (int)syscall(SYS_mbind, address, size, + MPOL_BIND | MPOL_F_STATIC_NODES, node_mask, + (unsigned long)numa_id + 1ul, flags); + free(node_mask); + return syscall_result == 0; +#else + (void)address; + (void)size; + (void)numa_id; + (void)flags; + return 0; +#endif +} + +static int sn_linux_populate_writable_mapping(void *address, size_t size) +{ +#if defined(MADV_POPULATE_WRITE) + if (madvise(address, size, MADV_POPULATE_WRITE) == 0) + return 1; + + return errno == EINVAL; +#else + (void)address; + (void)size; + return 1; +#endif +} + /* Pin the calling thread to a single logical processor. * core_id >= 0 : bind to that logical processor (global index, across groups). * core_id < 0 : clear affinity constraint within the current group. */ SN_API void SN_DECL sn_set_thread_affinity(int core_id) { - long ncpus = sysconf(_SC_NPROCESSORS_ONLN); + int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN); + if (cpu_count <= 0) + return; + + cpu_set_t *set = CPU_ALLOC(cpu_count); + size_t size = CPU_ALLOC_SIZE(cpu_count); + if (set == NULL) + return; - cpu_set_t *set = CPU_ALLOC(ncpus); - size_t size = CPU_ALLOC_SIZE(ncpus); CPU_ZERO_S(size, set); if (core_id < 0) { - for (long i = 0; i < ncpus; i++) + for (int i = 0; i < cpu_count; i++) CPU_SET_S((int)i, size, set); } else { CPU_SET_S(core_id, size, set); @@ -31,16 +242,89 @@ SN_API void SN_DECL sn_set_thread_affinity(int core_id) SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa) { - (void)out_numa_count; - (void)count_of_each_numa; - return 0; + char buffer[256]; + if (!sn_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer))) + return 0; + + sn_linux_max_context max_context = {-1}; + if (!sn_linux_visit_id_list(buffer, sn_linux_track_max_id, &max_context) || + max_context.max_value < 0) + return 0; + + if (out_numa_count != NULL) + *out_numa_count = max_context.max_value + 1; + + if (count_of_each_numa != NULL) { + for (int index = 0; index <= max_context.max_value; index++) + count_of_each_numa[index] = 0; + + sn_linux_fill_node_count_context fill_context = {count_of_each_numa}; + if (!sn_linux_visit_id_list(buffer, sn_linux_fill_node_cpu_count, &fill_context)) + return 0; + } + + return 1; } -SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *in_numa_cpuid, uint64_t *global_cpuid) +SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global) +{ + if (numa_id < 0) + return 0; + + char path[128]; + char buffer[256]; + snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id); + if (!sn_linux_read_first_line(path, buffer, sizeof(buffer))) + return 0; + + sn_linux_fill_cpu_context fill_context = {internal, global, 0}; + return sn_linux_visit_id_list(buffer, sn_linux_fill_cpu_id, &fill_context); +} + +SN_API void *SN_DECL sn_numa_alloc(size_t size, int numa_id) +{ + size_t allocation_size = sn_linux_round_up_to_page_size(size); + if (allocation_size == 0) + return NULL; + + int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS; +#if defined(MAP_POPULATE) + if (numa_id >= 0) + mmap_flags |= MAP_POPULATE; +#endif + + void *memory = mmap(NULL, allocation_size, PROT_READ | PROT_WRITE, + mmap_flags, -1, 0); + if (memory == MAP_FAILED) + return NULL; + + if (numa_id < 0) + return memory; + + uint32_t bind_flags = 0u; +#if defined(MAP_POPULATE) + bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT; +#endif + + if (!sn_linux_node_exists(numa_id) || + !sn_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) || + !sn_linux_populate_writable_mapping(memory, allocation_size)) { + munmap(memory, allocation_size); + return NULL; + } + + return memory; +} + +SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id) { (void)numa_id; - (void)in_numa_cpuid; - (void)global_cpuid; - return 0; + + if (ptr == NULL || size == 0) + return; + + size_t allocation_size = sn_linux_round_up_to_page_size(size); + if (allocation_size != 0) + munmap(ptr, allocation_size); } #endif diff --git a/StgSharp.Native/sn-src/thread.windows.c b/StgSharp.Native/sn-src/thread.windows.c index 27a0326e..378a1894 100644 --- a/StgSharp.Native/sn-src/thread.windows.c +++ b/StgSharp.Native/sn-src/thread.windows.c @@ -56,14 +56,41 @@ SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *g int intra = 0; while (mask) { - uint32_t idx; + DWORD idx; _BitScanForward64(&idx, mask); - int _ = (internal == NULL) ? intra : (internal[intra] = intra); - global[intra] = (int)idx; + if (internal != NULL) + internal[intra] = (uint64_t)intra; + global[intra] = (uint32_t)idx; intra++; mask &= mask - 1; } return 1; } + +SN_API void *SN_DECL sn_numa_alloc(size_t size, int numa_id) +{ + if (size == 0) + return NULL; + + if (numa_id < 0) + return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + + ULONG highest = 0; + if (!GetNumaHighestNodeNumber(&highest) || (ULONG)numa_id > highest) + return NULL; + + return VirtualAllocExNuma(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT, + PAGE_READWRITE, (DWORD)numa_id); +} + +SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id) +{ + (void)size; + (void)numa_id; + + if (ptr != NULL) + VirtualFree(ptr, 0, MEM_RELEASE); +} + #endif