diff --git a/StgSharp.Common/Collections/StringHashMultiplexer.cs b/StgSharp.Common/Collections/StringHashMultiplexer.cs deleted file mode 100644 index a0225adf..00000000 --- a/StgSharp.Common/Collections/StringHashMultiplexer.cs +++ /dev/null @@ -1,162 +0,0 @@ -//----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -// file="StringHashMultiplexer" -// Project: StgSharp -// AuthorGroup: Nitload -// Copyright (c) Nitload. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -// -// ----------------------------------------------------------------------- -// ----------------------------------------------------------------------- -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading; - -namespace StgSharp.Collections -{ - public sealed class StringHashMultiplexer : IHashMultiplexer - { - - private int _initCount, _remainedCount, _hashCache, _hashLast; - private string _source, _sourceLast; - - public int RemainedMultiplexCount - { - get => _remainedCount; - set => _initCount = value; - } - - public void CacheHash(string obj, int count) - { - _hashCache = ForceComputeHash(obj); - _initCount = count; - _remainedCount = _initCount; - _source = obj; - } - - public bool Equals(string x, string y) - { - return x == y; - } - - public void ExtendCache(int count) - { - _initCount += count; - _remainedCount += count; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe int ForceComputeHash([NotNull] string obj) - { - int hash; - fixed (char* cPtr = obj.AsSpan()) { - hash = NativeIntrinsic.Context.city_hash_simplify(cPtr, obj.Length); - } - return hash; - } - - public int GetHashCode(string obj) - { - if (_remainedCount > 0 && ReferenceEquals(_source, obj)) - { - _remainedCount--; - return _hashCache; - } - if (ReferenceEquals(_sourceLast, obj)) { - return _hashLast; - } - _sourceLast = obj; - _hashLast = ForceComputeHash(obj); - return _hashLast; - } - - } - - public sealed class ConcurrentStringHashMultiplexer : IHashMultiplexer, IDisposable - { - - private bool disposedValue; - private readonly ThreadLocal _multiplexers = - new(() => new StringHashMultiplexer()); - - public int RemainedMultiplexCount - { - get => _multiplexers.Value!.RemainedMultiplexCount; - set => _multiplexers.Value!.RemainedMultiplexCount = value; - } - - public void CacheHash(string obj, int count) - { - throw new NotImplementedException(); - } - - // ~ConcurrentStringHashMultiplexer() - // { - // Dispose(disposing: false); - // } - - public void Dispose() - { - Dispose(disposing:true); - GC.SuppressFinalize(this); - } - - public bool Equals(string x, string y) - { - return x == y; - } - - public void ExtendCache(int count) - { - _multiplexers.Value!.ExtendCache(count); - } - - public unsafe int ForceComputeHash([NotNull] string obj) - { - int hash; - fixed (char* cPtr = obj.AsSpan()) { - hash = NativeIntrinsic.Context.city_hash_simplify(cPtr, obj.Length); - } - return hash; - } - - public int GetHashCode(string obj) - { - return _multiplexers.Value!.GetHashCode(obj); - } - - private void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) { - _multiplexers.Dispose(); - } - - disposedValue = true; - } - } - - } -} diff --git a/StgSharp.Common/Collections/SwissTable.X86.cs b/StgSharp.Common/Collections/SwissTable.X86.cs new file mode 100644 index 00000000..9ed3cef9 --- /dev/null +++ b/StgSharp.Common/Collections/SwissTable.X86.cs @@ -0,0 +1,178 @@ +//----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +// file="SwissTable.X86" +// Project: StgSharp +// AuthorGroup: Nitload +// Copyright (c) Nitload. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ----------------------------------------------------------------------- +// ----------------------------------------------------------------------- +using System.Numerics; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; + +namespace StgSharp.Collections +{ + public partial class SwissTable + { + + private unsafe class BucketScannerX86 : IBucketScanner + { + + public bool TryFind( + nint key, + uint Hash, + Bucket* bucket, + out nint value + ) + { + byte ctrl = (byte)(Hash >> 8); + Vector128 ctrlVector = Vector128.Create(ctrl); + for (int i = 0; i < BucketSize; i += 16) + { + Vector128 bucketCtrlVector = Sse2.LoadVector128(&bucket->Control.Control + i); + Vector128 cmpResult = Sse2.CompareEqual(ctrlVector, bucketCtrlVector); + int mask = Sse2.MoveMask(cmpResult); + while (mask != 0) + { + int bitPos = BitOperations.TrailingZeroCount(mask); + int index = i + bitPos; + if (bucket->Keys.value == key) + { + value = bucket->Values.value; + return true; + } + mask &= mask - 1; + } + } + value = default; + return false; + } + + public bool TryRemove( + nint key, + uint hash, + Bucket* bucket, + out nint value + ) + { + byte ctrl = (byte)(hash >> 8); + Vector128 ctrlVector = Vector128.Create(ctrl); + + for (int i = 0; i < BucketSize; i += 16) + { + Vector128 bucketCtrl = Sse2.LoadVector128( + &bucket->Control[i]); + Vector128 matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl); + int matches = Sse2.MoveMask(matchMask); + + while (matches != 0) + { + int bitPos = BitOperations.TrailingZeroCount(matches); + int index = i + bitPos; + if (bucket->Keys[index] == key) + { + value = bucket->Values[index]; + bucket->Control[index] = 0xFE; // mark as DELETED + // no need to remove Key/Value + return true; + } + matches &= matches - 1; + } + } + value = default; + return false; + } + + public bool TrySet( + nint key, + uint hash, + Bucket* bucket, + nint value, + out bool isNewKey + ) + { + byte ctrl = (byte)(hash >> 8); // H2: upper 8 bits (highest bit is naturally 0) + Vector128 ctrlVector = Vector128.Create(ctrl); + + int firstAvailableSlot = -1; // Record the first insertable slot + + for (int i = 0; i < BucketSize; i += 16) + { + Vector128 bucketCtrl = Sse2.LoadVector128( + &bucket->Control[i]); + + // === Phase 1: Look up an existing key === + Vector128 matchMask = Sse2.CompareEqual(ctrlVector, bucketCtrl); + int matches = Sse2.MoveMask(matchMask); + + while (matches != 0) + { + int bitPos = BitOperations.TrailingZeroCount(matches); + int index = i + bitPos; + if (bucket->Keys[index] == key) + { + // Key already exists -> update value + bucket->Values[index] = value; + isNewKey = false; + return true; + } + matches &= matches - 1; + } + + // === Phase 2: Track an available insertion slot === + if (firstAvailableSlot == -1) + { + // Find EMPTY (0xFF) or DELETED (0xFE) + // Trick: bytes with high bit set are insertable + int emptyOrDeleted = Sse2.MoveMask(bucketCtrl); + + // MoveMask extracts the top bit of each byte + // EMPTY/DELETED top bit = 1 -> mask bit = 1 + // OCCUPIED top bit = 0 -> mask bit = 0 + + if (emptyOrDeleted != 0) + { + int bitPos = BitOperations.TrailingZeroCount(emptyOrDeleted); + firstAvailableSlot = i + bitPos; + } + } + } + + // === Phase 3: Key not found, try to insert === + if (firstAvailableSlot == -1) + { + // Bucket is full; resize is required + isNewKey = false; + return false; + } + + bucket->Control[firstAvailableSlot] = ctrl; + bucket->Keys[firstAvailableSlot] = key; + bucket->Values[firstAvailableSlot] = value; + isNewKey = true; + return true; + } + + } + + } +} diff --git a/StgSharp.Common/Collections/SwissTable.cs b/StgSharp.Common/Collections/SwissTable.cs index b9f62859..69132917 100644 --- a/StgSharp.Common/Collections/SwissTable.cs +++ b/StgSharp.Common/Collections/SwissTable.cs @@ -26,34 +26,170 @@ // ----------------------------------------------------------------------- // ----------------------------------------------------------------------- using System; -using System.Collections.Generic; -using System.Linq; +using System.Numerics; using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; namespace StgSharp.Collections { - public unsafe class SwissTable + /// + /// + /// + public unsafe partial class SwissTable { - public bool TryGet(nint key, out nint value) + + private const int BucketCount = 1 << EntryMaskWidth; + private const int BucketSize = 256; + private const int EntryMaskWidth = 10; // 1024 entries total, 256 per bucket + public const int MaxCapacity = 1024 * BucketSize; // 256 entries per bucket, 1024 buckets + + private readonly Entry* _entries; + + private SwissTable( + Entry* buffer + ) + { + _entries = buffer; + for (int i = 0; i < BucketCount; i++) + { + // Set all control bytes to 0xFF (empty) + Unsafe.InitBlock(&_entries[i].Bucket.Control, 0xFF, BucketSize); + } + } + + public bool TryAddOrSet( + nint key, + nint value + ) { throw new NotImplementedException(); + } + + public bool TryAddOrSet( + nint key, + ref nint value, + out bool isNewKey + ) + { + uint hash = BitOperations.Crc32C(0xffff9961, (ulong)key); + short entry = (short)(hash & ((1 << 11) - 1)); + Bucket* bucket = &_entries[entry].Bucket; + return IBucketScanner.Current.TrySet(key, hash, bucket, value, out isNewKey); + } + + public bool TryGet( + nint key, + out nint value + ) + { + uint hash = BitOperations.Crc32C(0xffff9961, (ulong)key); + short entry = (short)(hash & ((1 << 11) - 1)); + Bucket* bucket = &_entries[entry].Bucket; + return IBucketScanner.Current.TryFind(key, hash, bucket, out value); /* * hash = CRC32 of key * entry = low 8 of hash * ctrl = 8-15 of hash * index = rest of hash * find value of hash - */ + */ } + public bool TryRemove( + nint key, + out nint value + ) + { + uint hash = BitOperations.Crc32C(0xffff9961, (ulong)key); + short entry = (short)(hash & ((1 << 11) - 1)); + Bucket* bucket = &_entries[entry].Bucket; + return IBucketScanner.Current.TryRemove(key, hash, bucket, out value); + } + + [InlineArray(BucketCount)] private struct Entry { - public fixed byte entry[256]; //entry is fully occupied, or empty, or deleted - // array of ctrl array - // array of value array - // array of hash array + + public Bucket Bucket; + } + + private unsafe struct Bucket + { + + public BucketControl Control; + public BucketKeyStorage Keys; + public BucketValueStorage Values; + + } + + private interface IBucketScanner + { + + static IBucketScanner Current { get; } = Create(); + + bool TryFind( + nint key, + uint Hash, + Bucket* bucket, + out nint value + ); + + unsafe bool TryRemove( + nint key, + uint hash, + Bucket* bucket, + out nint value + ); + + bool TrySet( + nint key, + uint hash, + Bucket* bucket, + nint value, + out bool isNewKey + ); + + private static IBucketScanner Create() + { + if (X86Base.IsSupported) + { + return new BucketScannerX86(); + } else if (ArmBase.IsSupported) + { + throw new NotSupportedException("ARM architecture is currently not supported for bucket scanning. Please use an x86 platform."); + } else + { + throw new NotSupportedException("No suitable bucket scanner implementation found for the current platform."); + } + } + + } + + [InlineArray(BucketSize)] + private struct BucketControl + { + + public byte Control; + + } + + [InlineArray(BucketSize)] + private struct BucketKeyStorage + { + + public nint value; + + } + + [InlineArray(BucketSize)] + private struct BucketValueStorage + { + + public nint value; + + } + } -} +} \ No newline at end of file diff --git a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h index 91555b44..0b97e260 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.context.x86.h @@ -182,15 +182,15 @@ SN_MK_PROC_DECL_STD(double, avx, , fma); SN_MK_PROC_DECL_STD(double, avx, _fma, fma); SN_MK_PROC_DECL_STD(double, 512, , fma); -SN_MK_PROC_DECL_STD(float, sse, , fma); -SN_MK_PROC_DECL_STD(float, avx, , fma); -SN_MK_PROC_DECL_STD(float, avx, _fma, fma); -SN_MK_PROC_DECL_STD(float, 512, , fma); +SN_MK_PROC_DECL_STD(float, sse, , tiled_fma); +SN_MK_PROC_DECL_STD(float, avx, , tiled_fma); +SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma); +SN_MK_PROC_DECL_STD(float, 512, , tiled_fma); -SN_MK_PROC_DECL_STD(double, sse, , fma); -SN_MK_PROC_DECL_STD(double, avx, , fma); -SN_MK_PROC_DECL_STD(double, avx, _fma, fma); -SN_MK_PROC_DECL_STD(double, 512, , fma); +SN_MK_PROC_DECL_STD(double, sse, , tiled_fma); +SN_MK_PROC_DECL_STD(double, avx, , tiled_fma); +SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma); +SN_MK_PROC_DECL_STD(double, 512, , tiled_fma); #pragma endregion diff --git a/StgSharp.Native/sn-head/sn_intrinsic.h b/StgSharp.Native/sn-head/sn_intrinsic.h index 77496e3c..b51a9c04 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.h @@ -54,12 +54,12 @@ typedef union scalar_pack { MAT_KERNEL(T) * mat_1; /* Offset 8: 2nd matrix kernel base */ \ MAT_KERNEL(T) * mat_2; /* Offset 16: 3rd matrix kernel base */ \ void *scalar; /* Offset 24: scalar pack */ \ - int32_t i; /* Offset 32: row index of the result tile */ \ - int32_t j; /* Offset 36: column index of the result tile */ \ - int32_t k; /* Offset 40: column index of the result tile */ \ - int32_t common_i; /* Offset 44: row index of the result tile */ \ - int32_t common_j; /* Offset 48: column index of the result tile */ \ - int32_t common_k; /* Offset 52: column index of the result tile */ \ + int32_t x; /* Offset 32: row index of the result tile */ \ + int32_t y; /* Offset 36: column index of the result tile */ \ + int32_t z; /* Offset 40: column index of the result tile */ \ + int32_t common_x; /* Offset 44: row index of the result tile */ \ + int32_t common_y; /* Offset 48: column index of the result tile */ \ + int32_t common_z; /* Offset 52: column index of the result tile */ \ uint32_t mask; /* Offset 56: mask for task schedualing, not useful in C side */ \ uint32_t reverse; /* Offset 60: reserved for future use */ \ scalar_pack padding; /* Offset 64: padding to make the struct size 128 bytes */ \ diff --git a/StgSharp.Native/sn-head/sn_intrinsic.std.h b/StgSharp.Native/sn-head/sn_intrinsic.std.h index 1cd4c27e..bee3f3f8 100644 --- a/StgSharp.Native/sn-head/sn_intrinsic.std.h +++ b/StgSharp.Native/sn-head/sn_intrinsic.std.h @@ -30,28 +30,28 @@ #define _SN_MATPROC_RIGHT_ANS_SCALAR 1 #define _SN_MATPROC_ANS_SCALAR 1 -#define _SN_MK_EXPAND_RIGHT_ANS \ - _mat_kernel_local const *right = mat_task_local->mat_1; \ - _mat_kernel_local *restrict ans = mat_task_local->mat_2; +#define _SN_MK_EXPAND_RIGHT_ANS \ + SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \ + SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; -#define _SN_MK_EXPAND_LEFT_RIGHT_ANS(T) \ - _mat_kernel_local const *left = mat_task_local->mat_0; \ - _mat_kernel_local const *right = mat_task_local->mat_1; \ - _mat_kernel_local *restrict ans = mat_task_local->mat_2; +#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \ + SN_MAT_KERNEL_LOCAL const *left = mat_task_local->mat_0; \ + SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \ + SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; -#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR(T) \ - _mat_kernel_local const *right = mat_task_local->mat_1; \ - _mat_kernel_local *restrict ans = mat_task_local->mat_2; \ +#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \ + SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \ + SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \ scalar_pack *restrict scalar = mat_task_local->scalar; -#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR(T) \ - _mat_kernel_local const *left = mat_task_local->mat_0; \ - _mat_kernel_local const *right = mat_task_local->mat_1; \ - _mat_kernel_local *restrict ans = mat_task_local->mat_2; \ +#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \ + SN_MAT_KERNEL_LOCAL const *left = mat_task_local->mat_0; \ + SN_MAT_KERNEL_LOCAL const *right = mat_task_local->mat_1; \ + SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \ scalar_pack *restrict scalar = mat_task_local->scalar; -#define _SN_MK_EXPAND_ANS_SCALAR(T) \ - MAT_KERNEL(T) *restrict ans = mat_task_local->mat_2; \ +#define _SN_MK_EXPAND_ANS_SCALAR \ + SN_MAT_KERNEL_LOCAL *restrict ans = mat_task_local->mat_2; \ scalar_pack *restrict scalar = &(mat_task_local->padding); #define _SN_PARAM_STYLE(T) _SN_MATPROC_##T @@ -70,11 +70,17 @@ size_t count = UNSAFE_AS(size_t, task->padding.data[1]); #define _MK_MKPROC_EXPAND_INDEX_KERNEL \ - size_t i = UNSAFE_AS(size_t, task->i); \ - size_t j = UNSAFE_AS(size_t, task->j); \ - size_t common_i = UNSAFE_AS(size_t, task->common_i); \ - size_t common_j = UNSAFE_AS(size_t, task->common_j); \ - size_t common_k = UNSAFE_AS(size_t, task->common_k); + size_t x = UNSAFE_AS(size_t, task->x); \ + size_t y = UNSAFE_AS(size_t, task->y); \ + size_t common_x = UNSAFE_AS(size_t, task->common_x); \ + size_t common_y = UNSAFE_AS(size_t, task->common_y); \ + size_t common_z = UNSAFE_AS(size_t, task->common_z); +#define _MK_MKPROC_EXPAND_INDEX_TILE \ + size_t x = UNSAFE_AS(size_t, task->x); \ + size_t y = UNSAFE_AS(size_t, task->y); \ + size_t common_x = UNSAFE_AS(size_t, task->common_x); \ + size_t common_y = UNSAFE_AS(size_t, task->common_y); \ + size_t common_z = UNSAFE_AS(size_t, task->common_z); #define _SN_MK_EXPAND_MEM(mem) _MK_MKPROC_EXPAND_INDEX_##mem #define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T @@ -82,7 +88,7 @@ #pragma endregion #define __mk_param_std(mem, param, T) \ - typedef MAT_KERNEL(T) _mat_kernel_local; \ + typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \ typedef MAT_TASK(T) _mat_task_local; \ _mat_task_local *_ = (_SN_PARAM_STYLE(param), (_SN_MEM_LAYOUT(mem)), task); \ _SN_MK_EXPAND_PARAM(param); \ diff --git a/StgSharp.Native/sn-src/check.ps1 b/StgSharp.Native/sn-src/check.ps1 new file mode 100644 index 00000000..611fa5de --- /dev/null +++ b/StgSharp.Native/sn-src/check.ps1 @@ -0,0 +1,25 @@ +# 1. 版本 — 已确认 18.1.8 +clang-format --version + +# 2. 不可见字符扫描 — 用完整路径 +$file = Join-Path $PWD "simdid.x86.c" +$bytes = [System.IO.File]::ReadAllBytes($file) + +for ($i = 0; $i -lt $bytes.Length - 2; $i++) { + if ($bytes[$i] -eq 0xEF -and $bytes[$i+1] -eq 0xBB -and $bytes[$i+2] -eq 0xBF) { + Write-Host "BOM found at offset $i" + } + if ($bytes[$i] -eq 0xC2 -and $bytes[$i+1] -eq 0xA0) { + Write-Host "NBSP at offset $i" + } + if ($bytes[$i] -eq 0xE2 -and $bytes[$i+1] -eq 0x80 -and + ($bytes[$i+2] -eq 0x8B -or $bytes[$i+2] -eq 0x8C -or $bytes[$i+2] -eq 0x8D)) { + Write-Host "Zero-width char at offset $i" + } +} + +Write-Host "Scan complete. File size: $($bytes.Length) bytes" + +# 3. 格式化对比 — 用 fc.exe 避免 PowerShell 的 Format-Custom 歧义 +clang-format -assume-filename=test.c $file > formatted.c +fc.exe $file formatted.c \ No newline at end of file diff --git a/StgSharp.Native/sn-src/formatted.c b/StgSharp.Native/sn-src/formatted.c new file mode 100644 index 00000000..84dacc9c --- /dev/null +++ b/StgSharp.Native/sn-src/formatted.c @@ -0,0 +1,372 @@ +#include "sn_target.h" +#include "sn_intrinsic.h" +#include "StgSharpNative.h" + +static SIMDID global_simdid = { 0 }; + +#if SN_IS_ARCH(SN_ARCH_X86_64) + +#include "sn_intrinsic.context.x86.h" + +#include +#include +#if defined(_MSC_VER) +#include +#else +#include +#endif + +static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx) +{ +#if defined(_MSC_VER) + int regs[4]; + __cpuidex(regs, leaf, subleaf); + *eax = regs[0]; + *ebx = regs[1]; + *ecx = regs[2]; + *edx = regs[3]; +#else + unsigned int a, b, c, d; + __cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d); + *eax = (int)a; + *ebx = (int)b; + *ecx = (int)c; + *edx = (int)d; +#endif +} + +static int xgetbv_has_xmm_ymm(void) +{ +#if defined(_MSC_VER) + unsigned __int64 xcr0 = _xgetbv(0); +#else + unsigned long long xcr0 = __builtin_ia32_xgetbv(0); +#endif + return ((xcr0 & 0x6) == 0x6); +} + +static int xgetbv_has_avx512_state(void) +{ + const unsigned long long mask = (1ull << 0) | (1ull << 1) | (1ull << 2) | (1ull << 5) | + (1ull << 6); +#if defined(_MSC_VER) + unsigned __int64 xcr0 = _xgetbv(0); +#else + unsigned long long xcr0 = __builtin_ia32_xgetbv(0); +#endif + return ((xcr0 & mask) == mask); +} + +static void decode_family_model(int cpuid1_eax, int *family, int *model) +{ + int family_value = 0; + int model_value = 0; + const int base_fam = (cpuid1_eax >> 8) & 0xF; + const int ext_fam = (cpuid1_eax >> 20) & 0xFF; + const int base_model = (cpuid1_eax >> 4) & 0xF; + const int ext_model = (cpuid1_eax >> 16) & 0xF; + + if (base_fam == 0xF) { + family_value = base_fam + ext_fam; + } else { + family_value = base_fam; + } + + if ((base_fam == 0x6) || (base_fam == 0xF)) { + model_value = (ext_model << 4) | base_model; + } else { + model_value = base_model; + } + + *family = family_value; + *model = model_value; +} + +/* AVX-512 execution-width classification for AMD, by CPU family. + * Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL. + * Zen5+ (>=0x1A): native 512-bit execution units → NATIVE. + * Other/future AMD families → UNKNOWN (conservative). */ +static uint64_t amd_avx512_impl(int family) +{ + if (family == 0x19) { + return SIMDID_AVX512_IMPL_DUAL; + } else if (family >= 0x1A) { + return SIMDID_AVX512_IMPL_NATIVE; + } else { + return SIMDID_AVX512_IMPL_UNKNOWN; + } +} + +/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units). + * - AMD Zen1 (family 0x17, model < 0x30) and Zen+ (0x18-0x2F) use a split/glued path. + * - Hygon Dhyana is architecturally Zen1-based; always split. + * - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */ +static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax) +{ + int family = 0; + int model = 0; + + if (strcmp(vendor, "HygonGenuine") == 0) { + return 1; + } + if (strcmp(vendor, "AuthenticAMD") != 0) { + return 0; + } + + decode_family_model(cpuid1_eax, &family, &model); + return (family == 0x17) && (model < 0x30); +} + +SN_API SIMDID SN_DECL sn_get_simd_level_local() +{ + SIMDID simdid = { 0 }; + + int eax = 0; + int ebx = 0; + int ecx = 0; + int edx = 0; + int family = 0; + int model = 0; + + get_cpuid(0, 0, &eax, &ebx, &ecx, &edx); + const int max_basic_leaf = eax; + + char vendor[13]; + memcpy(vendor + 0, &ebx, 4); + memcpy(vendor + 4, &edx, 4); + memcpy(vendor + 8, &ecx, 4); + vendor[12] = '\0'; + + const int is_intel = (strcmp(vendor, "GenuineIntel") == 0); + const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0); + const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); + const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0); + const int is_zhaoxin = (memcmp(vendor, " Shanghai ", 12) == 0); + const int is_via = is_centaur || is_zhaoxin; + + get_cpuid(1, 0, &eax, &ebx, &ecx, &edx); + const int cpuid1_eax = eax; + decode_family_model(cpuid1_eax, &family, &model); + (void)model; + + const int has_sse42 = (ecx & (1 << 20)) != 0; + const int has_avx = (ecx & (1 << 28)) != 0; + const int has_osxsave = (ecx & (1 << 27)) != 0; + const int has_fma = (ecx & (1 << 12)) != 0; + const int has_f16c = (ecx & (1 << 29)) != 0; + + const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); + uint64_t manu_code = SIMDID_MANU_UNKNOWN; + if (is_intel) { + manu_code = SIMDID_MANU_INTEL; + } else if (is_amd) { + manu_code = SIMDID_MANU_AMD; + } else if (is_hygon) { + manu_code = SIMDID_MANU_HYGON; + } else if (is_via) { + manu_code = SIMDID_MANU_VIA; + } + const uint64_t manu = SIMDID_PACK_MANU(manu_code); + + uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE); + uint64_t avx_bits = 0; + uint64_t avx512_bits = 0; + uint64_t amx_bits = 0; + uint64_t avx10_bits = 0; + uint64_t uarch_bits = 0; + int has_hybrid = 0; + + if (has_sse42) { + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE); + } + + if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) { + avx_bits |= SIMDID_AVX; + + if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) { + avx_bits |= SIMDID_AVX_DUAL; + } + + if (max_basic_leaf >= 7) { + get_cpuid(7, 0, &eax, &ebx, &ecx, &edx); + + const int has_avx2 = (ebx & (1 << 5)) != 0; + const int has_avx512f = (ebx & (1 << 16)) != 0; + const int has_avx512dq = (ebx & (1 << 17)) != 0; + const int has_avx512bw = (ebx & (1 << 30)) != 0; + const int has_avx512vl = (ebx & (1u << 31)) != 0; + const int has_avx512_vnni = (ecx & (1 << 11)) != 0; + const int has_vbmi = (ecx & (1 << 1)) != 0; + const int has_vbmi2 = (ecx & (1 << 6)) != 0; + + const int has_amx_tile = (edx & (1 << 24)) != 0; + const int has_amx_int8 = (edx & (1 << 25)) != 0; + const int has_amx_bf16 = (edx & (1 << 22)) != 0; + has_hybrid = (edx & (1 << 15)) != 0; + + int has_bf16 = 0; + int has_fp16 = 0; + int has_avx10 = 0; + if (eax >= 1) { + get_cpuid(7, 1, &eax, &ebx, &ecx, &edx); + has_bf16 = (eax & (1 << 5)) != 0; + has_fp16 = (eax & (1 << 23)) != 0; + has_avx10 = (edx & (1 << 19)) != 0; + } + + if (has_avx2) { + avx_bits |= SIMDID_AVX2; + if (has_fma) { + avx_bits |= SIMDID_AVX_FMA; + } + if (has_f16c) { + avx_bits |= SIMDID_AVX_F16C; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2); + } + + if (has_avx512f && xgetbv_has_avx512_state()) { + if (has_avx512bw && has_avx512vl && has_avx512dq) { + avx512_bits |= SIMDID_AVX512_BASE; + } + if (has_avx512_vnni) { + avx512_bits |= SIMDID_AVX512_VNNI; + } + if (has_bf16) { + avx512_bits |= SIMDID_AVX512_BF16; + } + if (has_fp16) { + avx512_bits |= SIMDID_AVX512_FP16; + } + if (has_vbmi) { + avx512_bits |= SIMDID_AVX512_VBMI; + } + if (has_vbmi2) { + avx512_bits |= SIMDID_AVX512_VBMI2; + } + + if (is_intel) { + avx512_bits |= SIMDID_AVX512_IMPL_NATIVE; + } else if (is_amd) { + avx512_bits |= amd_avx512_impl(family); + } else { + avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN; + } + + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512); + + if (has_amx_tile) { + amx_bits |= SIMDID_AMX_TILE; + if (has_amx_int8) { + amx_bits |= SIMDID_AMX_INT8; + } + if (has_amx_bf16) { + amx_bits |= SIMDID_AMX_BF16; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX); + } + + if (has_avx10 && (max_basic_leaf >= 0x24)) { + int a10_eax = 0; + int a10_ebx = 0; + int a10_ecx = 0; + int a10_edx = 0; + int avx10_ver = 0; + int has_256w = 0; + int has_512w = 0; + + get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx); + avx10_ver = a10_ebx & 0xFF; + if (avx10_ver >= 1) { + avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1; + if (avx10_ver >= 2) { + avx10_bits |= SIMDID_AVX10_V2; + } + has_256w = (a10_ebx & (1 << 17)) != 0; + has_512w = (a10_ebx & (1 << 18)) != 0; + if (has_256w) { + avx10_bits |= SIMDID_AVX10_256W; + } + if (has_512w) { + avx10_bits |= SIMDID_AVX10_512W; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10); + } + } + } + } + } + + if (has_hybrid && (max_basic_leaf >= 0x1A)) { + int h_eax = 0; + int h_ebx = 0; + int h_ecx = 0; + int h_edx = 0; + + get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx); + { + const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU; + if (core_type == 0x40U) { + uarch_bits |= SIMDID_UARCH_HYBRID_P; + } else if (core_type == 0x20U) { + uarch_bits |= SIMDID_UARCH_HYBRID_E; + } + } + } else { + uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E; + } + + { + const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0; + const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK; + const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL; + if (has_avx_dual || has_avx512_dual) { + uarch_bits |= SIMDID_UARCH_SIMD_DUAL; + } + } + + { + uint64_t final_mask = root; + final_mask |= manu; + final_mask |= main_lvl; + final_mask |= SIMDID_PACK_AVX512(avx512_bits); + final_mask |= SIMDID_PACK_AMX(amx_bits); + final_mask |= SIMDID_PACK_AVX(avx_bits); + final_mask |= SIMDID_PACK_AVX10(avx10_bits); + final_mask |= SIMDID_PACK_UARCH(uarch_bits); + simdid.mask = final_mask; + } + + return simdid; +} + +SN_API SIMDID SN_DECL sn_get_simd_level_global() +{ + if (global_simdid.mask) { + return global_simdid; + } + + global_simdid = sn_get_simd_level_local(); + return global_simdid; +} + +#else + +SN_API SIMDID SN_DECL sn_get_simd_level_local() +{ + SIMDID simdid; + simdid.mask = 0; + return simdid; +} + +SN_API SIMDID SN_DECL sn_get_simd_level_global() +{ + if (global_simdid.mask) { + return global_simdid; + } + + global_simdid = sn_get_simd_level_local(); + return global_simdid; +} + +#endif diff --git a/StgSharp.Native/sn-src/load_intrinsic.x86.c b/StgSharp.Native/sn-src/load_intrinsic.x86.c index 746bc383..6ea2fdb3 100644 --- a/StgSharp.Native/sn-src/load_intrinsic.x86.c +++ b/StgSharp.Native/sn-src/load_intrinsic.x86.c @@ -58,7 +58,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, sse, , sub); context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, sse, , transpose); context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, sse, , fma); - context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , fma); + context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, sse, , tiled_fma); context->mat[F32].pivot = NULL; context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, sse, , add); @@ -67,7 +67,7 @@ void load_intrinsic_sse(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, sse, , sub); context->mat[F64].buffer_transpose = NULL; context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, sse, , fma); - context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , fma); + context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, sse, , tiled_fma); context->mat[F64].pivot = NULL; context->vec_f32_normalize_3 = f32_normalize; @@ -90,8 +90,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, avx, , transpose); context->mat[F32].ker_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) : STD_MK_PROC_NAME(float, avx, _fma, fma); - context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , fma) : - STD_MK_PROC_NAME(float, avx, _fma, fma); + context->mat[F32].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(float, avx, , tiled_fma) : + STD_MK_PROC_NAME(float, avx, _fma, tiled_fma); context->mat[F32].pivot = NULL; context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, avx, , add); @@ -101,8 +101,8 @@ void load_intrinsic_avx(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_transpose = NULL; context->mat[F64].ker_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) : STD_MK_PROC_NAME(double, avx, _fma, fma); - context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , fma) : - STD_MK_PROC_NAME(double, avx, _fma, fma); + context->mat[F64].ker_tile_fma = is_fma ? STD_MK_PROC_NAME(double, avx, , tiled_fma) : + STD_MK_PROC_NAME(double, avx, _fma, tiled_fma); context->mat[F64].pivot = NULL; context->vec_f32_normalize_3 = f32_normalize; @@ -120,7 +120,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id) context->mat[F32].buffer_sub = STD_MK_PROC_NAME(float, 512, , sub); context->mat[F32].buffer_transpose = STD_MK_PROC_NAME(float, 512, , transpose); context->mat[F32].ker_fma = STD_MK_PROC_NAME(float, 512, , fma); - context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , fma); + context->mat[F32].ker_tile_fma = STD_MK_PROC_NAME(float, 512, , tiled_fma); context->mat[F32].pivot = NULL; context->mat[F64].buffer_add = STD_MK_PROC_NAME(double, 512, , add); @@ -129,7 +129,7 @@ void load_intrinsic_512(sn_intrinsic *context, SIMDID id) context->mat[F64].buffer_sub = STD_MK_PROC_NAME(double, 512, , sub); context->mat[F64].buffer_transpose = NULL; context->mat[F64].ker_fma = STD_MK_PROC_NAME(double, 512, , fma); - context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , fma); + context->mat[F64].ker_tile_fma = STD_MK_PROC_NAME(double, 512, , tiled_fma); context->mat[F64].pivot = NULL; context->vec_f32_normalize_3 = f32_normalize; diff --git a/StgSharp.Native/sn-src/m.fma_tile.x86.c b/StgSharp.Native/sn-src/m.fma_tile.x86.c index 631ba2bd..ba4f1ae1 100644 --- a/StgSharp.Native/sn-src/m.fma_tile.x86.c +++ b/StgSharp.Native/sn-src/m.fma_tile.x86.c @@ -12,17 +12,12 @@ // float -SN_MK_PROC_DECL_STD(float, sse, , fma) +SN_MK_PROC_DECL_STD(float, sse, , tiled_fma) { - __mk_param_std(); - - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, float); MAT_KERNEL(float) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m128 c0 = _mm_load_ps((float *)&ans_cur->f32_x[0]); register __m128 c1 = _mm_load_ps((float *)&ans_cur->f32_x[1]); @@ -30,11 +25,9 @@ SN_MK_PROC_DECL_STD(float, sse, , fma) register __m128 c3 = _mm_load_ps((float *)&ans_cur->f32_x[3]); register __m128 bcol; - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(float) - const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe - const *left_cur = - left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); const __m128 a0 = _mm_load_ps((float *)&left_cur->f32_x[0]); const __m128 a1 = _mm_load_ps((float *)&left_cur->f32_x[1]); @@ -76,18 +69,15 @@ SN_MK_PROC_DECL_STD(float, sse, , fma) _mm_store_ps((float *)&ans_cur->f32_x[3], c3); } -SN_MK_PROC_DECL_STD(float, avx, , fma) +SN_MK_PROC_DECL_STD(float, avx, , tiled_fma) { static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, float); MAT_KERNEL(float) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m256 c01 = _mm256_load_ps((float *)&ans->f32_y[0]); register __m256 c23 = _mm256_load_ps((float *)&ans->f32_y[1]); @@ -95,11 +85,9 @@ SN_MK_PROC_DECL_STD(float, avx, , fma) register __m256i b01_shift = _mm256_load_si256(&b01_shift_source); register __m256i b23_shift = _mm256_load_si256(&b23_shift_source); - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(float) - const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe - const *right_cur = - right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]); register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]); @@ -139,18 +127,15 @@ SN_MK_PROC_DECL_STD(float, avx, , fma) _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); } -SN_MK_PROC_DECL_STD(float, avx, _fma, fma) +SN_MK_PROC_DECL_STD(float, avx, _fma, tiled_fma) { static const __m256i b01_shift_source = { .m256i_i32 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m256i b23_shift_source = { .m256i_i32 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, float); MAT_KERNEL(float) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m256 c01 = _mm256_load_ps((float *)&ans_cur->f32_y[0]); register __m256 c23 = _mm256_load_ps((float *)&ans_cur->f32_y[1]); @@ -158,11 +143,9 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) register __m256i b01_shift = _mm256_load_si256(&b01_shift_source); register __m256i b23_shift = _mm256_load_si256(&b23_shift_source); - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(float) - const *left_cur = left + GET_INDEX(ansHeight, k, j), // GetKernelAddressUnsafe - const *right_cur = - right + GET_INDEX(commonK, i, k); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); register __m256 a01 = _mm256_load_ps((float *)&left_cur->f32_y[0]); register __m256 a23 = _mm256_load_ps((float *)&left_cur->f32_y[1]); @@ -202,7 +185,7 @@ SN_MK_PROC_DECL_STD(float, avx, _fma, fma) _mm256_store_ps((float *)&ans_cur->f32_y[1], c23); } -SN_MK_PROC_DECL_STD(float, 512, , fma) +SN_MK_PROC_DECL_STD(float, 512, , tiled_fma) { static const __m512i b_shift_source = { .m512i_i32 = { // shuff order @@ -211,18 +194,13 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) 10, 11, 8, 9, //col2 15, 12, 13, 14 //col3 } }; - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; - register __m512i b_shift = _mm512_load_si512(&b_shift_source); - register __m512 c = _mm512_load_ps((float *)(ans + GET_INDEX(ansHeight, i, j))); - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(double) *const left_cur = left + GET_INDEX(ansHeight, k, j), - const *right_cur = - right + GET_INDEX(commonK, i, - k); // GetKernelAddressUnsafe + __mk_param_std(TILE, LEFT_RIGHT_ANS, float); + SN_MAT_KERNEL_LOCAL const *ans_cur = ans + GET_INDEX(common_y, x, y); + register __m512 c = _mm512_load_ps(ans_cur); + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); register __m512 a = _mm512_load_ps((float *)left_cur); register __m512 b = _mm512_load_ps((float *)right_cur); @@ -245,7 +223,7 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) c = _mm512_fmadd_ps(a_perm, b_perm, c); } - _mm512_store_ps((float *)(ans + GET_INDEX(ansHeight, i, j)), c); + _mm512_store_ps((float *)ans_cur, c); } // double @@ -282,26 +260,25 @@ SN_MK_PROC_DECL_STD(float, 512, , fma) c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \ } while (0) -SN_MK_PROC_DECL_STD(double, sse, , fma) +SN_MK_PROC_DECL_STD(double, sse, , tiled_fma) { - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, double); - register __m128d c0_lo = _mm_load_pd(ans->f64_x + (0 * 2 + 0)); - register __m128d c0_hi = _mm_load_pd(ans->f64_x + (0 * 2 + 1)); - register __m128d c1_lo = _mm_load_pd(ans->f64_x + (1 * 2 + 0)); - register __m128d c1_hi = _mm_load_pd(ans->f64_x + (1 * 2 + 1)); - register __m128d c2_lo = _mm_load_pd(ans->f64_x + (2 * 2 + 0)); - register __m128d c2_hi = _mm_load_pd(ans->f64_x + (2 * 2 + 1)); - register __m128d c3_lo = _mm_load_pd(ans->f64_x + (3 * 2 + 0)); - register __m128d c3_hi = _mm_load_pd(ans->f64_x + (3 * 2 + 1)); + MAT_KERNEL(double) + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(double) - const *left_cur = left + GET_INDEX(ansHeight, i, k), - const *right_cur = right + GET_INDEX(commonK, j, k); // GetKernelAddressUnsafe + register __m128d c0_lo = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 0)); + register __m128d c0_hi = _mm_load_pd(ans_cur->f64_x + (0 * 2 + 1)); + register __m128d c1_lo = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 0)); + register __m128d c1_hi = _mm_load_pd(ans_cur->f64_x + (1 * 2 + 1)); + register __m128d c2_lo = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 0)); + register __m128d c2_hi = _mm_load_pd(ans_cur->f64_x + (2 * 2 + 1)); + register __m128d c3_lo = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 0)); + register __m128d c3_hi = _mm_load_pd(ans_cur->f64_x + (3 * 2 + 1)); + + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); DOUBLE_KERNEL_FMA_CYCLE(0); @@ -312,25 +289,22 @@ SN_MK_PROC_DECL_STD(double, sse, , fma) DOUBLE_KERNEL_FMA_CYCLE(3); } - _mm_store_pd(ans->f64_x + (0 * 2 + 0), c0_lo); - _mm_store_pd(ans->f64_x + (0 * 2 + 1), c0_hi); - _mm_store_pd(ans->f64_x + (1 * 2 + 0), c1_lo); - _mm_store_pd(ans->f64_x + (1 * 2 + 1), c1_hi); - _mm_store_pd(ans->f64_x + (2 * 2 + 0), c2_lo); - _mm_store_pd(ans->f64_x + (2 * 2 + 1), c2_hi); - _mm_store_pd(ans->f64_x + (3 * 2 + 0), c3_lo); - _mm_store_pd(ans->f64_x + (3 * 2 + 1), c3_hi); + _mm_store_pd(ans_cur->f64_x + (0 * 2 + 0), c0_lo); + _mm_store_pd(ans_cur->f64_x + (0 * 2 + 1), c0_hi); + _mm_store_pd(ans_cur->f64_x + (1 * 2 + 0), c1_lo); + _mm_store_pd(ans_cur->f64_x + (1 * 2 + 1), c1_hi); + _mm_store_pd(ans_cur->f64_x + (2 * 2 + 0), c2_lo); + _mm_store_pd(ans_cur->f64_x + (2 * 2 + 1), c2_hi); + _mm_store_pd(ans_cur->f64_x + (3 * 2 + 0), c3_lo); + _mm_store_pd(ans_cur->f64_x + (3 * 2 + 1), c3_hi); } -SN_MK_PROC_DECL_STD(double, avx, , fma) +SN_MK_PROC_DECL_STD(double, avx, , tiled_fma) { - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, double); MAT_KERNEL(double) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]); register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]); @@ -338,11 +312,9 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]); register __m256d bcol; - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(double) - const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe - const *left_cur = - left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]); const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]); @@ -400,15 +372,12 @@ SN_MK_PROC_DECL_STD(double, avx, , fma) _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); } -SN_MK_PROC_DECL_STD(double, avx, _fma, fma) +SN_MK_PROC_DECL_STD(double, avx, _fma, tiled_fma) { - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, double); MAT_KERNEL(double) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m256d c0 = _mm256_load_pd((double *)&ans_cur->f64_y[0]); register __m256d c1 = _mm256_load_pd((double *)&ans_cur->f64_y[1]); @@ -416,11 +385,9 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma) register __m256d c3 = _mm256_load_pd((double *)&ans_cur->f64_y[3]); register __m256d bcol; - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(double) - const *right_cur = right + GET_INDEX(commonK, i, k), // GetKernelAddressUnsafe - const *left_cur = - left + GET_INDEX(ansHeight, k, j); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); const __m256d a0 = _mm256_load_pd((double *)&left_cur->f64_y[0]); const __m256d a1 = _mm256_load_pd((double *)&left_cur->f64_y[1]); @@ -462,18 +429,15 @@ SN_MK_PROC_DECL_STD(double, avx, _fma, fma) _mm256_store_pd((double *)&ans_cur->f64_y[3], c3); } -SN_MK_PROC_DECL_STD(double, 512, , fma) +SN_MK_PROC_DECL_STD(double, 512, , tiled_fma) { static const __m512i b01_shift_source = { .m512i_i64 = { 0, 1, 2, 3, 5, 6, 7, 4 } }; static const __m512i b23_shift_source = { .m512i_i64 = { 2, 3, 0, 1, 7, 4, 5, 6 } }; - int i = enumeration->m128_i32[0]; - int j = enumeration->m128_i32[1]; - int commonK = enumeration->m128_i32[2]; - int ansHeight = enumeration->m128_i32[3]; + __mk_param_std(TILE, LEFT_RIGHT_ANS, double); MAT_KERNEL(double) - const *ans_cur = ans + GET_INDEX(ansHeight, i, j); // GetKernelAddressUnsafe + const *ans_cur = ans + GET_INDEX(common_y, x, y); // GetKernelAddressUnsafe register __m512d c01 = _mm512_load_pd((double *)&ans->f64_z[0]); register __m512d c23 = _mm512_load_pd((double *)&ans->f64_z[1]); @@ -481,11 +445,9 @@ SN_MK_PROC_DECL_STD(double, 512, , fma) register __m512i b01_shift = _mm512_load_si512(&b01_shift_source); register __m512i b23_shift = _mm512_load_si512(&b23_shift_source); - for (size_t k = 0; k < commonK; k++) { - MAT_KERNEL(double) - const *left_cur = left + GET_INDEX(ansHeight, i, k), // GetKernelAddressUnsafe - const *right_cur = - right + GET_INDEX(ansHeight, j, k); // GetKernelAddressUnsafe + for (size_t z = 0; z < common_z; z++) { + SN_MAT_KERNEL_LOCAL const *right_cur = right + GET_INDEX(common_z, x, z), + const *left_cur = left + GET_INDEX(common_y, z, y); register __m512d a01 = _mm512_load_pd((double *)&left_cur->f64_z[0]); register __m512d a23 = _mm512_load_pd((double *)&left_cur->f64_z[1]); diff --git a/StgSharp.Native/sn-src/m.solve.x86.c b/StgSharp.Native/sn-src/m.solve.x86.c index 373511d2..2e307674 100644 --- a/StgSharp.Native/sn-src/m.solve.x86.c +++ b/StgSharp.Native/sn-src/m.solve.x86.c @@ -10,6 +10,8 @@ static const float quality_threshold = 1e-6f; SN_MK_PROC_DECL_STD(float, sse, , try_plu) { + __mk_param_std(TILE, LEFT_RIGHT_ANS_SCALAR, float); + /* Aliases */ MAT_KERNEL(float) *upper = left; /* U */ MAT_KERNEL(float) *lower = right; /* L (unit diag) */ diff --git a/StgSharp.Native/sn-src/simdid.x86.c b/StgSharp.Native/sn-src/simdid.x86.c index b852d787..84dacc9c 100644 --- a/StgSharp.Native/sn-src/simdid.x86.c +++ b/StgSharp.Native/sn-src/simdid.x86.c @@ -6,7 +6,7 @@ static SIMDID global_simdid = { 0 }; #if SN_IS_ARCH(SN_ARCH_X86_64) -#include "sn_intrincontext.x86.h" +#include "sn_intrinsic.context.x86.h" #include #include @@ -57,15 +57,44 @@ static int xgetbv_has_avx512_state(void) return ((xcr0 & mask) == mask); } +static void decode_family_model(int cpuid1_eax, int *family, int *model) +{ + int family_value = 0; + int model_value = 0; + const int base_fam = (cpuid1_eax >> 8) & 0xF; + const int ext_fam = (cpuid1_eax >> 20) & 0xFF; + const int base_model = (cpuid1_eax >> 4) & 0xF; + const int ext_model = (cpuid1_eax >> 16) & 0xF; + + if (base_fam == 0xF) { + family_value = base_fam + ext_fam; + } else { + family_value = base_fam; + } + + if ((base_fam == 0x6) || (base_fam == 0xF)) { + model_value = (ext_model << 4) | base_model; + } else { + model_value = base_model; + } + + *family = family_value; + *model = model_value; +} + /* AVX-512 execution-width classification for AMD, by CPU family. * Zen4 (0x19): single 256-bit ALU, 512-bit op = two 256-bit micro-ops (dual-pump) → DUAL. * Zen5+ (>=0x1A): native 512-bit execution units → NATIVE. * Other/future AMD families → UNKNOWN (conservative). */ static uint64_t amd_avx512_impl(int family) { - if (family == 0x19) return SIMDID_AVX512_IMPL_DUAL; - if (family >= 0x1A) return SIMDID_AVX512_IMPL_NATIVE; - return SIMDID_AVX512_IMPL_UNKNOWN; + if (family == 0x19) { + return SIMDID_AVX512_IMPL_DUAL; + } else if (family >= 0x1A) { + return SIMDID_AVX512_IMPL_NATIVE; + } else { + return SIMDID_AVX512_IMPL_UNKNOWN; + } } /* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units). @@ -74,194 +103,241 @@ static uint64_t amd_avx512_impl(int family) * - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */ static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax) { - if (strcmp(vendor, "HygonGenuine") == 0) - return 1; /* Hygon is Zen1-based; always split YMM */ - if (strcmp(vendor, "AuthenticAMD") != 0) + int family = 0; + int model = 0; + + if (strcmp(vendor, "HygonGenuine") == 0) { + return 1; + } + if (strcmp(vendor, "AuthenticAMD") != 0) { return 0; - int base_fam = (cpuid1_eax >> 8) & 0xF; - int ext_fam = (cpuid1_eax >> 20) & 0xFF; - int disp_fam = (base_fam < 0xF) ? base_fam : (base_fam + ext_fam); - int base_mdl = (cpuid1_eax >> 4) & 0xF; - int ext_mdl = (cpuid1_eax >> 16) & 0xF; - int disp_mdl = (ext_mdl << 4) | base_mdl; - return (disp_fam == 0x17) && (disp_mdl < 0x30); /* Zen1/Zen+ */ + } + + decode_family_model(cpuid1_eax, &family, &model); + return (family == 0x17) && (model < 0x30); } SN_API SIMDID SN_DECL sn_get_simd_level_local() { - SIMDID _simdid = { 0 }; + SIMDID simdid = { 0 }; - int eax = 0, ebx = 0, ecx = 0, edx = 0; + int eax = 0; + int ebx = 0; + int ecx = 0; + int edx = 0; + int family = 0; + int model = 0; - /* Vendor */ get_cpuid(0, 0, &eax, &ebx, &ecx, &edx); + const int max_basic_leaf = eax; + char vendor[13]; - ((int *)vendor)[0] = ebx; - ((int *)vendor)[1] = edx; - ((int *)vendor)[2] = ecx; + memcpy(vendor + 0, &ebx, 4); + memcpy(vendor + 4, &edx, 4); + memcpy(vendor + 8, &ecx, 4); vendor[12] = '\0'; + const int is_intel = (strcmp(vendor, "GenuineIntel") == 0); - const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0); - const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); /* 海光: separate from AMD */ - const int is_via = (strcmp(vendor, "CentaurHauls") == 0) || - (memcmp(vendor, " Shanghai ", 12) == 0); /* VIA / Zhaoxin */ + const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0); + const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0); + const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0); + const int is_zhaoxin = (memcmp(vendor, " Shanghai ", 12) == 0); + const int is_via = is_centaur || is_zhaoxin; - /* CPUID.1 */ get_cpuid(1, 0, &eax, &ebx, &ecx, &edx); - int cpuid1_eax = eax; /* saved for Zen1/Zen+ glue-YMM detection */ - int family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF0); - int model = ((eax >> 4) & 0xF) | ((eax >> 12) & 0xF0); + const int cpuid1_eax = eax; + decode_family_model(cpuid1_eax, &family, &model); + (void)model; - int has_sse42 = (ecx & (1 << 20)) != 0; - int has_avx = (ecx & (1 << 28)) != 0; - int has_osxsave = (ecx & (1 << 27)) != 0; - int has_fma = (ecx & (1 << 12)) != 0; - int has_f16c = (ecx & (1 << 29)) != 0; + const int has_sse42 = (ecx & (1 << 20)) != 0; + const int has_avx = (ecx & (1 << 28)) != 0; + const int has_osxsave = (ecx & (1 << 27)) != 0; + const int has_fma = (ecx & (1 << 12)) != 0; + const int has_f16c = (ecx & (1 << 29)) != 0; + + const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); + uint64_t manu_code = SIMDID_MANU_UNKNOWN; + if (is_intel) { + manu_code = SIMDID_MANU_INTEL; + } else if (is_amd) { + manu_code = SIMDID_MANU_AMD; + } else if (is_hygon) { + manu_code = SIMDID_MANU_HYGON; + } else if (is_via) { + manu_code = SIMDID_MANU_VIA; + } + const uint64_t manu = SIMDID_PACK_MANU(manu_code); - uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64); - uint64_t manu = SIMDID_PACK_MANU(is_intel ? SIMDID_MANU_INTEL : - is_amd ? SIMDID_MANU_AMD : - is_hygon ? SIMDID_MANU_HYGON : - is_via ? SIMDID_MANU_VIA : - SIMDID_MANU_UNKNOWN); uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE); + uint64_t avx_bits = 0; uint64_t avx512_bits = 0; uint64_t amx_bits = 0; - uint64_t avx_bits = 0; uint64_t avx10_bits = 0; - int has_hybrid = 0; /* filled from CPUID.7.0 EDX[15] when AVX present */ + uint64_t uarch_bits = 0; + int has_hybrid = 0; - /* Baseline main level */ - if (has_sse42) + if (has_sse42) { main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE); + } if (has_avx && has_osxsave && xgetbv_has_xmm_ymm()) { avx_bits |= SIMDID_AVX; - /* Glued/split-lane YMM: early AMD Zen1/Zen+, Hygon, and VIA/Zhaoxin */ - if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) + if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) { avx_bits |= SIMDID_AVX_DUAL; - - /* CPUID.7.0 */ - get_cpuid(7, 0, &eax, &ebx, &ecx, &edx); - const int has_bmi1 = (ebx & (1 << 3)) != 0; - const int has_bmi2 = (ebx & (1 << 8)) != 0; - const int has_avx2 = (ebx & (1 << 5)) != 0; - const int has_avx512f = (ebx & (1 << 16)) != 0; - const int has_avx512dq = (ebx & (1 << 17)) != 0; - const int has_avx512bw = (ebx & (1 << 30)) != 0; - const int has_avx512vl = (ebx & (1u << 31)) != 0; - const int has_avx512_vnni = (ebx & (1 << 11)) != 0; - const int has_vbmi = (ecx & (1 << 1)) != 0; - const int has_vbmi2 = (ecx & (1 << 6)) != 0; - - /* AMX in leaf 7 subleaf 0 EDX bits 24/25/26 — capture before - overwriting regs with subleaf 1 */ - const int has_amx_tile = (edx & (1 << 24)) != 0; - const int has_amx_int8 = (edx & (1 << 25)) != 0; - const int has_amx_bf16 = (edx & (1 << 22)) != 0; - has_hybrid = (edx & (1 << 15)) != 0; /* Intel hybrid core topology */ - - /* leaf 7, subleaf 1 for BF16/FP16/AVX10 convergence */ - get_cpuid(7, 1, &eax, &ebx, &ecx, &edx); - const int has_bf16 = (eax & (1 << 5)) != 0; /* AVX512_BF16 */ - const int has_fp16 = (eax & (1 << 23)) != 0; /* AVX512_FP16 */ - const int has_avx10 = (edx & (1 << 19)) != 0; /* AVX10 convergence */ - - if (has_avx2) { - avx_bits |= SIMDID_AVX2; - if (has_fma) - avx_bits |= SIMDID_AVX_FMA; - if (has_f16c) - avx_bits |= SIMDID_AVX_F16C; - - main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2); } - if (has_avx512f && xgetbv_has_avx512_state()) { - /* Ensure base subset present */ - if (has_avx512bw && has_avx512vl && has_avx512dq) { - avx512_bits |= SIMDID_AVX512_BASE; - } - if (has_avx512_vnni) - avx512_bits |= SIMDID_AVX512_VNNI; - if (has_bf16) - avx512_bits |= SIMDID_AVX512_BF16; - if (has_fp16) - avx512_bits |= SIMDID_AVX512_FP16; - if (has_vbmi) - avx512_bits |= SIMDID_AVX512_VBMI; - if (has_vbmi2) - avx512_bits |= SIMDID_AVX512_VBMI2; + if (max_basic_leaf >= 7) { + get_cpuid(7, 0, &eax, &ebx, &ecx, &edx); - /* Execution-width classification: - * Intel all → NATIVE (genuine 512-bit ALU; SKX dual-port is native) - * AMD Zen4 → DUAL (dual-pump 256-bit, Family 0x19) - * AMD Zen5+ → NATIVE (native 512-bit, Family >= 0x1A) - * Hygon/VIA/unknown → UNKNOWN (conservative) */ - if (is_intel) - avx512_bits |= SIMDID_AVX512_IMPL_NATIVE; - else if (is_amd) - avx512_bits |= amd_avx512_impl(family); - else - avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN; + const int has_avx2 = (ebx & (1 << 5)) != 0; + const int has_avx512f = (ebx & (1 << 16)) != 0; + const int has_avx512dq = (ebx & (1 << 17)) != 0; + const int has_avx512bw = (ebx & (1 << 30)) != 0; + const int has_avx512vl = (ebx & (1u << 31)) != 0; + const int has_avx512_vnni = (ecx & (1 << 11)) != 0; + const int has_vbmi = (ecx & (1 << 1)) != 0; + const int has_vbmi2 = (ecx & (1 << 6)) != 0; - /* main level for AVX512 */ - main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512); + const int has_amx_tile = (edx & (1 << 24)) != 0; + const int has_amx_int8 = (edx & (1 << 25)) != 0; + const int has_amx_bf16 = (edx & (1 << 22)) != 0; + has_hybrid = (edx & (1 << 15)) != 0; - /* AMX promotion if present */ - if (has_amx_tile) { - amx_bits |= SIMDID_AMX_TILE; - if (has_amx_int8) - amx_bits |= SIMDID_AMX_INT8; - if (has_amx_bf16) - amx_bits |= SIMDID_AMX_BF16; - main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX); + int has_bf16 = 0; + int has_fp16 = 0; + int has_avx10 = 0; + if (eax >= 1) { + get_cpuid(7, 1, &eax, &ebx, &ecx, &edx); + has_bf16 = (eax & (1 << 5)) != 0; + has_fp16 = (eax & (1 << 23)) != 0; + has_avx10 = (edx & (1 << 19)) != 0; } - /* AVX10 detection — requires AVX10 convergence bit - (leaf 7 subleaf 1 EDX[19]) and leaf 0x24 */ - if (has_avx10) { - int a10_eax = 0, a10_ebx = 0, a10_ecx = 0, a10_edx = 0; - get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx); - const int avx10_ver = a10_ebx & 0xFF; - if (avx10_ver >= 1) { - avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1; - if (avx10_ver >= 2) - avx10_bits |= SIMDID_AVX10_V2; - if (a10_ebx & (1 << 17)) - avx10_bits |= SIMDID_AVX10_256W; - if (a10_ebx & (1 << 18)) - avx10_bits |= SIMDID_AVX10_512W; - main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10); + if (has_avx2) { + avx_bits |= SIMDID_AVX2; + if (has_fma) { + avx_bits |= SIMDID_AVX_FMA; + } + if (has_f16c) { + avx_bits |= SIMDID_AVX_F16C; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2); + } + + if (has_avx512f && xgetbv_has_avx512_state()) { + if (has_avx512bw && has_avx512vl && has_avx512dq) { + avx512_bits |= SIMDID_AVX512_BASE; + } + if (has_avx512_vnni) { + avx512_bits |= SIMDID_AVX512_VNNI; + } + if (has_bf16) { + avx512_bits |= SIMDID_AVX512_BF16; + } + if (has_fp16) { + avx512_bits |= SIMDID_AVX512_FP16; + } + if (has_vbmi) { + avx512_bits |= SIMDID_AVX512_VBMI; + } + if (has_vbmi2) { + avx512_bits |= SIMDID_AVX512_VBMI2; + } + + if (is_intel) { + avx512_bits |= SIMDID_AVX512_IMPL_NATIVE; + } else if (is_amd) { + avx512_bits |= amd_avx512_impl(family); + } else { + avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN; + } + + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512); + + if (has_amx_tile) { + amx_bits |= SIMDID_AMX_TILE; + if (has_amx_int8) { + amx_bits |= SIMDID_AMX_INT8; + } + if (has_amx_bf16) { + amx_bits |= SIMDID_AMX_BF16; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX); + } + + if (has_avx10 && (max_basic_leaf >= 0x24)) { + int a10_eax = 0; + int a10_ebx = 0; + int a10_ecx = 0; + int a10_edx = 0; + int avx10_ver = 0; + int has_256w = 0; + int has_512w = 0; + + get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx); + avx10_ver = a10_ebx & 0xFF; + if (avx10_ver >= 1) { + avx10_bits |= SIMDID_AVX10 | SIMDID_AVX10_V1; + if (avx10_ver >= 2) { + avx10_bits |= SIMDID_AVX10_V2; + } + has_256w = (a10_ebx & (1 << 17)) != 0; + has_512w = (a10_ebx & (1 << 18)) != 0; + if (has_256w) { + avx10_bits |= SIMDID_AVX10_256W; + } + if (has_512w) { + avx10_bits |= SIMDID_AVX10_512W; + } + main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10); + } } } } } - /* uArchHi: hybrid core topology and SIMD-dual summary */ - uint64_t uarch_bits = 0; - if (has_hybrid) { - int h_eax = 0, h_ebx = 0, h_ecx = 0, h_edx = 0; + if (has_hybrid && (max_basic_leaf >= 0x1A)) { + int h_eax = 0; + int h_ebx = 0; + int h_ecx = 0; + int h_edx = 0; + get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx); - unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU; - if (core_type == 0x40) /* Intel Core / P-core → 10 */ - uarch_bits |= SIMDID_UARCH_HYBRID_P; - else if (core_type == 0x20) /* Intel Atom / E-core → 01 */ - uarch_bits |= SIMDID_UARCH_HYBRID_E; - /* else unknown hybrid core type → topology stays 00 */ + { + const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU; + if (core_type == 0x40U) { + uarch_bits |= SIMDID_UARCH_HYBRID_P; + } else if (core_type == 0x20U) { + uarch_bits |= SIMDID_UARCH_HYBRID_E; + } + } } else { - /* Confirmed non-hybrid (unified) → set both bits = 11 (numerically strongest) */ uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E; } - if ((avx_bits & SIMDID_AVX_DUAL) || - ((avx512_bits & SIMDID_AVX512_IMPL_MASK) == SIMDID_AVX512_IMPL_DUAL)) - uarch_bits |= SIMDID_UARCH_SIMD_DUAL; - _simdid.mask = root | manu | main_lvl | SIMDID_PACK_AVX512(avx512_bits) | - SIMDID_PACK_AMX(amx_bits) | SIMDID_PACK_AVX(avx_bits) | - SIMDID_PACK_AVX10(avx10_bits) | SIMDID_PACK_UARCH(uarch_bits); - return _simdid; + { + const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0; + const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK; + const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL; + if (has_avx_dual || has_avx512_dual) { + uarch_bits |= SIMDID_UARCH_SIMD_DUAL; + } + } + + { + uint64_t final_mask = root; + final_mask |= manu; + final_mask |= main_lvl; + final_mask |= SIMDID_PACK_AVX512(avx512_bits); + final_mask |= SIMDID_PACK_AMX(amx_bits); + final_mask |= SIMDID_PACK_AVX(avx_bits); + final_mask |= SIMDID_PACK_AVX10(avx10_bits); + final_mask |= SIMDID_PACK_UARCH(uarch_bits); + simdid.mask = final_mask; + } + + return simdid; } SN_API SIMDID SN_DECL sn_get_simd_level_global() @@ -269,17 +345,28 @@ SN_API SIMDID SN_DECL sn_get_simd_level_global() if (global_simdid.mask) { return global_simdid; } + global_simdid = sn_get_simd_level_local(); return global_simdid; } #else -SIMDID sn_get_simd_level() +SN_API SIMDID SN_DECL sn_get_simd_level_local() { - SIMDID simd_id; - simd_id.mask = 0; - return simd_id; /* non-x86: no SIMD level probing */ + SIMDID simdid; + simdid.mask = 0; + return simdid; +} + +SN_API SIMDID SN_DECL sn_get_simd_level_global() +{ + if (global_simdid.mask) { + return global_simdid; + } + + global_simdid = sn_get_simd_level_local(); + return global_simdid; } #endif