new mat proc std
This commit is contained in:
Youmu Liu
@@ -1,19 +1,37 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- ISA target：x64 + ARM64 -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup Label="ProjectConfigurations">
|
||||
<ProjectConfiguration Include="Debug|x64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="PerformanceDebug|x64">
|
||||
<Configuration>PerformanceDebug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
<ProjectConfiguration Include="Debug|ARM64">
|
||||
<Configuration>Debug</Configuration>
|
||||
<Platform>ARM64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|x64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="Release|ARM64">
|
||||
<Configuration>Release</Configuration>
|
||||
<Platform>ARM64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="PerformanceDebug|x64">
|
||||
<Configuration>PerformanceDebug</Configuration>
|
||||
<Platform>x64</Platform>
|
||||
</ProjectConfiguration>
|
||||
<ProjectConfiguration Include="PerformanceDebug|ARM64">
|
||||
<Configuration>PerformanceDebug</Configuration>
|
||||
<Platform>ARM64</Platform>
|
||||
</ProjectConfiguration>
|
||||
</ItemGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- Global Property -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<PropertyGroup Label="Globals">
|
||||
<VCProjectVersion>17.0</VCProjectVersion>
|
||||
<Keyword>Win32Proj</Keyword>
|
||||
@@ -23,21 +41,27 @@
|
||||
<ProjectName>StgSharp.Native</ProjectName>
|
||||
</PropertyGroup>
|
||||
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- Configuration -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- Debug（x64 + ARM64 共用） -->
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>true</UseDebugLibraries>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<WholeProgramOptimization>false</WholeProgramOptimization>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
|
||||
<!-- Release（x64 + ARM64 共用） -->
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
<CharacterSet>Unicode</CharacterSet>
|
||||
<PlatformToolset>v143</PlatformToolset>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='PerformanceDebug|x64'" Label="Configuration">
|
||||
<!-- PerformanceDebug（x64 + ARM64 共用） -->
|
||||
<PropertyGroup Condition="'$(Configuration)'=='PerformanceDebug'" Label="Configuration">
|
||||
<ConfigurationType>DynamicLibrary</ConfigurationType>
|
||||
<UseDebugLibraries>false</UseDebugLibraries>
|
||||
<WholeProgramOptimization>true</WholeProgramOptimization>
|
||||
@@ -49,111 +73,104 @@
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="Shared">
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<ImportGroup Condition="'$(Configuration)|$(Platform)'=='PerformanceDebug|x64'" Label="PropertySheets">
|
||||
<!-- PropertySheets（对所有 Configuration|Platform 生效） -->
|
||||
<ImportGroup Label="PropertySheets">
|
||||
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
|
||||
</ImportGroup>
|
||||
<PropertyGroup Label="UserMacros" />
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<OutDir>$(SolutionDir)build</OutDir>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- output folder: layered -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<PropertyGroup>
|
||||
<!-- 把 ARM64 → arm64，x64 保持 x64 -->
|
||||
<_ArchTag Condition="'$(Platform)'=='x64'">x64</_ArchTag>
|
||||
<_ArchTag Condition="'$(Platform)'=='ARM64'">arm64</_ArchTag>
|
||||
<!-- 输出到 build/win-<arch>/ -->
|
||||
<OutDir>$(SolutionDir)build\win-$(_ArchTag)\</OutDir>
|
||||
<IntDir>$(SolutionDir)obj\$(ProjectName)\$(Configuration)\$(Platform)\</IntDir>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Debug'">
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
<EnableClangTidyCodeAnalysis>false</EnableClangTidyCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<OutDir>$(SolutionDir)build</OutDir>
|
||||
<PropertyGroup Condition="'$(Configuration)'=='Release' Or '$(Configuration)'=='PerformanceDebug'">
|
||||
<GenerateManifest>false</GenerateManifest>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
<EnableClangTidyCodeAnalysis>false</EnableClangTidyCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='PerformanceDebug|x64'">
|
||||
<OutDir>$(SolutionDir)build</OutDir>
|
||||
<GenerateManifest>false</GenerateManifest>
|
||||
<RunCodeAnalysis>true</RunCodeAnalysis>
|
||||
<EnableClangTidyCodeAnalysis>false</EnableClangTidyCodeAnalysis>
|
||||
</PropertyGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- Compile options: common to all configurations -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemDefinitionGroup>
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<SDLCheck>true</SDLCheck>
|
||||
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<ConformanceMode>true</ConformanceMode>
|
||||
<LanguageStandard>Default</LanguageStandard>
|
||||
<LanguageStandard_C>stdc11</LanguageStandard_C>
|
||||
<InlineFunctionExpansion>Disabled</InlineFunctionExpansion>
|
||||
<ExpandAttributedSource>true</ExpandAttributedSource>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
<BrowseInformation>true</BrowseInformation>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
<!-- DLL 应该用 Windows，不是 Console -->
|
||||
<SubSystem>Windows</SubSystem>
|
||||
<AdditionalDependencies>opengl32.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<!-- Debug 特有 -->
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
|
||||
<ClCompile>
|
||||
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<InlineFunctionExpansion>Disabled</InlineFunctionExpansion>
|
||||
<ExpandAttributedSource>true</ExpandAttributedSource>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
<BrowseInformation>true</BrowseInformation>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
<Bscmake>
|
||||
<PreserveSbr>true</PreserveSbr>
|
||||
</Bscmake>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
|
||||
<!-- Release + PerformanceDebug 共用（优化开启） -->
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release' Or '$(Configuration)'=='PerformanceDebug'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<SDLCheck>true</SDLCheck>
|
||||
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<ConformanceMode>true</ConformanceMode>
|
||||
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
<LanguageStandard>Default</LanguageStandard>
|
||||
<LanguageStandard_C>stdc11</LanguageStandard_C>
|
||||
<CompileAs>CompileAsC</CompileAs>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<GenerateDebugInformation>false</GenerateDebugInformation>
|
||||
<ShowProgress>LinkVerbose</ShowProgress>
|
||||
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<!-- Release 独有（InlineFunctionExpansion） -->
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
|
||||
<ClCompile>
|
||||
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<GenerateDebugInformation>false</GenerateDebugInformation>
|
||||
<AdditionalDependencies>opengl32.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<ShowProgress>LinkVerbose</ShowProgress>
|
||||
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='PerformanceDebug|x64'">
|
||||
<ClCompile>
|
||||
<WarningLevel>Level3</WarningLevel>
|
||||
<FunctionLevelLinking>true</FunctionLevelLinking>
|
||||
<IntrinsicFunctions>true</IntrinsicFunctions>
|
||||
<SDLCheck>true</SDLCheck>
|
||||
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
|
||||
<ConformanceMode>true</ConformanceMode>
|
||||
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
|
||||
<AssemblerOutput>All</AssemblerOutput>
|
||||
<LanguageStandard>Default</LanguageStandard>
|
||||
<LanguageStandard_C>stdc11</LanguageStandard_C>
|
||||
<CompileAs>CompileAsC</CompileAs>
|
||||
<MultiProcessorCompilation>true</MultiProcessorCompilation>
|
||||
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
|
||||
</ClCompile>
|
||||
<Link>
|
||||
<SubSystem>Console</SubSystem>
|
||||
<EnableCOMDATFolding>true</EnableCOMDATFolding>
|
||||
<OptimizeReferences>true</OptimizeReferences>
|
||||
<GenerateDebugInformation>false</GenerateDebugInformation>
|
||||
<AdditionalDependencies>opengl32.lib;$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<ShowProgress>LinkVerbose</ShowProgress>
|
||||
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- glfw static lib -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup Condition="'$(Platform)'=='x64'">
|
||||
<Library Include="lib\glfw\glfw3.lib" />
|
||||
<Library Include="lib\glfw\glfw3_mt.lib" />
|
||||
</ItemGroup>
|
||||
<ItemGroup Condition="'$(Platform)'=='ARM64'">
|
||||
<Library Include="lib\glfw\glfw3.lib" />
|
||||
<Library Include="lib\glfw\glfw3_mt.lib" />
|
||||
</ItemGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- source files -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup>
|
||||
<ClCompile Include="sn-src\m.basic.x86.c" />
|
||||
<ClCompile Include="sn-src\dllmain.c" />
|
||||
@@ -163,9 +180,7 @@
|
||||
<ClCompile Include="sn-src\s.factorial.x86.c" />
|
||||
<ClCompile Include="sn-src\m.fma.x86.c" />
|
||||
<ClCompile Include="sn-src\s.hash.x86.c" />
|
||||
<ClCompile Include="sn-src\load_intrinsic.x86.c">
|
||||
<FileType>CCode</FileType>
|
||||
</ClCompile>
|
||||
<ClCompile Include="sn-src\load_intrinsic.x86.c" />
|
||||
<ClCompile Include="sn-src\load_glfw.c" />
|
||||
<ClCompile Include="sn-src\simdid.x86.c" />
|
||||
<ClCompile Include="sn-src\simdid_compare.x86.c" />
|
||||
@@ -175,6 +190,9 @@
|
||||
<ClCompile Include="sn-src\vector.c" />
|
||||
<ClCompile Include="sn-src\vector_dot.c" />
|
||||
</ItemGroup>
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<!-- header files -->
|
||||
<!-- ════════════════════════════════════════════════════════ -->
|
||||
<ItemGroup>
|
||||
<ClInclude Include="lib\glfw\glfw3.h" />
|
||||
<ClInclude Include="lib\glfw\glfw3native.h" />
|
||||
@@ -183,17 +201,14 @@
|
||||
<ClInclude Include="sn-head\glfw_function.h" />
|
||||
<ClInclude Include="sn-head\khrplatform.h" />
|
||||
<ClInclude Include="sn-head\sn_internal.h" />
|
||||
<ClInclude Include="sn-head\sn_intrincontext.x86.h" />
|
||||
<ClInclude Include="sn-head\sn_intrinsic.context.x86.h" />
|
||||
<ClInclude Include="sn-head\sn_intrinsic.std.h" />
|
||||
<ClInclude Include="sn-head\sn_intrinsic.h" />
|
||||
<ClInclude Include="sn-head\sn_matkernel.h" />
|
||||
<ClInclude Include="sn-head\sn_intrinsic.matkernel.h" />
|
||||
<ClInclude Include="sn-head\sn_target.h" />
|
||||
<ClInclude Include="sn-head\StgSharpNative.h" />
|
||||
<ClInclude Include="sn-head\wgl.h" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Library Include="lib\glfw\glfw3.lib" />
|
||||
<Library Include="lib\glfw\glfw3_mt.lib" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="naming.md" />
|
||||
<None Include="SIMDID.x86.md" />
|
||||
|
||||
@@ -89,7 +89,7 @@
|
||||
<ClInclude Include="sn-head\glfw_function.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="sn-head\sn_matkernel.h">
|
||||
<ClInclude Include="sn-head\sn_intrinsic.matkernel.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="lib\glfw\glfw3.h">
|
||||
@@ -104,12 +104,15 @@
|
||||
<ClInclude Include="sn-head\sn_target.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="sn-head\sn_intrincontext.x86.h">
|
||||
<ClInclude Include="sn-head\sn_intrinsic.context.x86.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="sn-head\sn_intrinsic.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
<ClInclude Include="sn-head\sn_intrinsic.std.h">
|
||||
<Filter>头文件</Filter>
|
||||
</ClInclude>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Library Include="lib\glfw\glfw3.lib">
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
#include <xmmintrin.h>
|
||||
|
||||
#define UNSAFE_AS(type, expr) *(type *)&(expr)
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define _CRT_SECURE_NO_WARNINGS 1
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
|
||||
+234
-232
@@ -1,232 +1,234 @@
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// SIMDID layout helpers (see SIMDID.x86.md)
|
||||
// bits [3..0] : Root ISA
|
||||
// bits [7..4] : Manufacture (ISA-specific)
|
||||
// bits [15..8] : MainLevel (monotonic capability)
|
||||
// bits [31..16]: AVX512 Feature
|
||||
// bits [39..32]: AMX Feature
|
||||
// bits [47..40]: AVX Feature
|
||||
// bits [55..48]: AVX10 Feature
|
||||
// bits [63..56]: uArchHi / Policy (optional)
|
||||
// -----------------------------------------------------------------------------
|
||||
#define SIMDID_SHIFT_ROOT 0
|
||||
#define SIMDID_SHIFT_MANU 4
|
||||
#define SIMDID_SHIFT_MAIN 8
|
||||
#define SIMDID_SHIFT_AVX512 16
|
||||
#define SIMDID_SHIFT_AMX 32
|
||||
#define SIMDID_SHIFT_AVX 40
|
||||
#define SIMDID_SHIFT_AVX10 48
|
||||
#define SIMDID_SHIFT_UARCH 56
|
||||
|
||||
#define SIMDID_MASK_ROOT 0xFULL
|
||||
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
|
||||
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
|
||||
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
|
||||
#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
|
||||
#define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX)
|
||||
#define SIMDID_MASK_AVX10 (0xFFULL << SIMDID_SHIFT_AVX10)
|
||||
#define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH)
|
||||
|
||||
// Root ISA (low nibble)
|
||||
#define SIMDID_ROOT_UNKNOWN 0x0
|
||||
#define SIMDID_ROOT_X86_64 0x1
|
||||
#define SIMDID_ROOT_AARCH64 0x2
|
||||
#define SIMDID_ROOT_PPC64LE 0x3
|
||||
#define SIMDID_ROOT_RISCV64 0x4
|
||||
#define SIMDID_ROOT_LOONGARCH 0x5
|
||||
|
||||
// Manufacture (x86-64 mapping; other ISAs may reinterpret)
|
||||
#define SIMDID_MANU_UNKNOWN 0x0
|
||||
#define SIMDID_MANU_INTEL 0x1
|
||||
#define SIMDID_MANU_AMD 0x2
|
||||
#define SIMDID_MANU_VIA 0x3
|
||||
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
|
||||
|
||||
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
|
||||
#define SIMDID_MAIN_LVL_NONE 0x00u
|
||||
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
|
||||
#define SIMDID_MAIN_LVL_AVX2 0x02u
|
||||
#define SIMDID_MAIN_LVL_AVX512 0x03u
|
||||
#define SIMDID_MAIN_LVL_AMX 0x04u
|
||||
#define SIMDID_MAIN_LVL_AVX10 0x05u
|
||||
|
||||
#ifndef SIMDID_AVX512_BASE
|
||||
#define SIMDID_AVX512_FP16 (1U << 13) /* relative bit 13 within [31..16] */
|
||||
#define SIMDID_AVX512_BF16 (1U << 12)
|
||||
#define SIMDID_AVX512_VNNI (1U << 11)
|
||||
#define SIMDID_AVX512_VBMI2 (1U << 10)
|
||||
#define SIMDID_AVX512_VBMI (1U << 9)
|
||||
#define SIMDID_AVX512_AI_RESERVED (1U << 8)
|
||||
#define SIMDID_AVX512_BASE (1U << 4)
|
||||
#define SIMDID_AVX512_IMPL_SHIFT 6
|
||||
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AVX_BASE
|
||||
#define SIMDID_AVX (1U << 0) /* relative bit 0 within [47..40] */
|
||||
#define SIMDID_AVX2 (1U << 1)
|
||||
#define SIMDID_AVX_FMA (1U << 2)
|
||||
#define SIMDID_AVX_F16C (1U << 3)
|
||||
#define SIMDID_AVX_DUAL (1U << 4) /* glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AMX_TILE
|
||||
#define SIMDID_AMX_TILE (1U << 0) /* relative bit 0 within [39..32] */
|
||||
#define SIMDID_AMX_INT8 (1U << 1)
|
||||
#define SIMDID_AMX_BF16 (1U << 2)
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AVX10_BASE
|
||||
#define SIMDID_AVX10 (1U << 0) /* relative bit 0 within [55..48] */
|
||||
#define SIMDID_AVX10_V1 (1U << 1)
|
||||
#define SIMDID_AVX10_V2 (1U << 2)
|
||||
#define SIMDID_AVX10_256W (1U << 3)
|
||||
#define SIMDID_AVX10_512W (1U << 4)
|
||||
#endif
|
||||
|
||||
/* uArchHi ([63..56]) bitmask — core topology and SIMD-width execution policy */
|
||||
#ifndef SIMDID_UARCH_HYBRID_P
|
||||
/* Topology 2-bit field (bits [1..0] of uArchHi byte):
|
||||
* 00 = unknown (undetected hybrid type or pre-CPUID.1A hardware)
|
||||
* 01 = E-core (efficiency/small core in a hybrid CPU)
|
||||
* 10 = P-core (performance/big core in a hybrid CPU)
|
||||
* 11 = unified (confirmed non-hybrid; numerically strongest)
|
||||
* Higher numeric value == stronger / higher-capability topology. */
|
||||
#define SIMDID_UARCH_HYBRID_E (1U << 0) /* E-core (efficiency/small) in a hybrid topology */
|
||||
#define SIMDID_UARCH_HYBRID_P (1U << 1) /* P-core (performance/big) in a hybrid topology */
|
||||
#define SIMDID_UARCH_SIMD_DUAL (1U << 2) /* SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
|
||||
#endif
|
||||
|
||||
// Pack helpers (define locally to avoid missing macros when included standalone)
|
||||
#ifndef SIMDID_PACK_ROOT
|
||||
#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
|
||||
#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
|
||||
#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
|
||||
#define SIMDID_PACK_AVX512(avx512_bits) \
|
||||
(((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
|
||||
#define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX)
|
||||
#define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX)
|
||||
#define SIMDID_PACK_AVX10(avx10_bits) (((uint64_t)(avx10_bits) & 0xFFULL) << SIMDID_SHIFT_AVX10)
|
||||
#define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH)
|
||||
#endif
|
||||
|
||||
/* Pre-shifted helpers for convenience when constructing SIMDID */
|
||||
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
|
||||
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
|
||||
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
|
||||
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
|
||||
#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
|
||||
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
|
||||
|
||||
#pragma region matix function
|
||||
|
||||
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , add);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose);
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose);
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose);
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, sse, , clear);
|
||||
DECLARE_PNL_PROC_ANS(float, avx, , clear);
|
||||
DECLARE_PNL_PROC_ANS(float, 512, , clear);
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, sse, , clear);
|
||||
DECLARE_PNL_PROC_ANS(double, avx, , clear);
|
||||
DECLARE_PNL_PROC_ANS(double, 512, , clear);
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality);
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
||||
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
||||
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region scaler_simd
|
||||
|
||||
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region dot_product
|
||||
|
||||
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans);
|
||||
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region string
|
||||
|
||||
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#else
|
||||
|
||||
typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
|
||||
|
||||
#endif
|
||||
#include "sn_target.h"
|
||||
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// SIMDID layout helpers (see SIMDID.x86.md)
|
||||
// bits [3..0] : Root ISA
|
||||
// bits [7..4] : Manufacture (ISA-specific)
|
||||
// bits [15..8] : MainLevel (monotonic capability)
|
||||
// bits [31..16]: AVX512 Feature
|
||||
// bits [39..32]: AMX Feature
|
||||
// bits [47..40]: AVX Feature
|
||||
// bits [55..48]: AVX10 Feature
|
||||
// bits [63..56]: uArchHi / Policy (optional)
|
||||
// -----------------------------------------------------------------------------
|
||||
#define SIMDID_SHIFT_ROOT 0
|
||||
#define SIMDID_SHIFT_MANU 4
|
||||
#define SIMDID_SHIFT_MAIN 8
|
||||
#define SIMDID_SHIFT_AVX512 16
|
||||
#define SIMDID_SHIFT_AMX 32
|
||||
#define SIMDID_SHIFT_AVX 40
|
||||
#define SIMDID_SHIFT_AVX10 48
|
||||
#define SIMDID_SHIFT_UARCH 56
|
||||
|
||||
#define SIMDID_MASK_ROOT 0xFULL
|
||||
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
|
||||
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
|
||||
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
|
||||
#define SIMDID_MASK_AMX (0xFFULL << SIMDID_SHIFT_AMX)
|
||||
#define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX)
|
||||
#define SIMDID_MASK_AVX10 (0xFFULL << SIMDID_SHIFT_AVX10)
|
||||
#define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH)
|
||||
|
||||
// Root ISA (low nibble)
|
||||
#define SIMDID_ROOT_UNKNOWN 0x0
|
||||
#define SIMDID_ROOT_X86_64 0x1
|
||||
#define SIMDID_ROOT_AARCH64 0x2
|
||||
#define SIMDID_ROOT_PPC64LE 0x3
|
||||
#define SIMDID_ROOT_RISCV64 0x4
|
||||
#define SIMDID_ROOT_LOONGARCH 0x5
|
||||
|
||||
// Manufacture (x86-64 mapping; other ISAs may reinterpret)
|
||||
#define SIMDID_MANU_UNKNOWN 0x0
|
||||
#define SIMDID_MANU_INTEL 0x1
|
||||
#define SIMDID_MANU_AMD 0x2
|
||||
#define SIMDID_MANU_VIA 0x3
|
||||
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
|
||||
|
||||
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
|
||||
#define SIMDID_MAIN_LVL_NONE 0x00u
|
||||
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
|
||||
#define SIMDID_MAIN_LVL_AVX2 0x02u
|
||||
#define SIMDID_MAIN_LVL_AVX512 0x03u
|
||||
#define SIMDID_MAIN_LVL_AMX 0x04u
|
||||
#define SIMDID_MAIN_LVL_AVX10 0x05u
|
||||
|
||||
#ifndef SIMDID_AVX512_BASE
|
||||
#define SIMDID_AVX512_FP16 U << 13) relative bit 13 within [31..16] */
|
||||
#define SIMDID_AVX512_BF16 U << 12)
|
||||
#define SIMDID_AVX512_VNNI U << 11)
|
||||
#define SIMDID_AVX512_VBMI2 U << 10)
|
||||
#define SIMDID_AVX512_VBMI U << 9)
|
||||
#define SIMDID_AVX512_AI_RESERVED U << 8)
|
||||
#define SIMDID_AVX512_BASE U << 4)
|
||||
#define SIMDID_AVX512_IMPL_SHIFT
|
||||
#define SIMDID_AVX512_IMPL_MASK x3U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_UNKNOWN x0U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_DUAL x1U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#define SIMDID_AVX512_IMPL_NATIVE x2U << SIMDID_AVX512_IMPL_SHIFT)
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AVX_BASE
|
||||
#define SIMDID_AVX U << 0) relative bit 0 within [47..40] */
|
||||
#define SIMDID_AVX2 U << 1)
|
||||
#define SIMDID_AVX_FMA U << 2)
|
||||
#define SIMDID_AVX_F16C (1U << 3)
|
||||
#define SIMDID_AVX_DUAL ( \
|
||||
U << 4) glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AMX_TILE
|
||||
#define SIMDID_AMX_TILE (1U << 0) relative bit 0 within [39..32] */
|
||||
#define SIMDID_AMX_INT8 (1U << 1)
|
||||
#define SIMDID_AMX_BF16 (1U << 2)
|
||||
#endif
|
||||
|
||||
#ifndef SIMDID_AVX10_BASE
|
||||
#define SIMDID_AVX10 U << 0) /* relative bit 0 within [55..48] */
|
||||
#define SIMDID_AVX10_V1 U << 1)
|
||||
#define SIMDID_AVX10_V2 U << 2)
|
||||
#define SIMDID_AVX10_256W (1U << 3)
|
||||
#define SIMDID_AVX10_512W (1U << 4)
|
||||
#endif
|
||||
|
||||
/* uArchHi ([63..56]) bitmask — core topology and SIMD-width execution policy */
|
||||
#ifndef SIMDID_UARCH_HYBRID_P
|
||||
/* Topology 2-bit field (bits [1..0] of uArchHi byte):
|
||||
* 00 = unknown (undetected hybrid type or pre-CPUID.1A hardware)
|
||||
* 01 = E-core (efficiency/small core in a hybrid CPU)
|
||||
* 10 = P-core (performance/big core in a hybrid CPU)
|
||||
* 11 = unified (confirmed non-hybrid; numerically strongest)
|
||||
* Higher numeric value == stronger / higher-capability topology. */
|
||||
#define SIMDID_UARCH_HYBRID_E << 0) / E-core (efficiency/small) in a hybrid topology */
|
||||
#define SIMDID_UARCH_HYBRID_P << 1) / P-core (performance/big) in a hybrid topology */
|
||||
#define SIMDID_UARCH_SIMD_DUAL ( \
|
||||
<< 2) / SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
|
||||
#endif
|
||||
|
||||
// Pack helpers (define locally to avoid missing macros when included standalone)
|
||||
#ifndef SIMDID_PACK_ROOT
|
||||
#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
|
||||
#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
|
||||
#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
|
||||
#define SIMDID_PACK_AVX512(avx512_bits) \
|
||||
(((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
|
||||
#define SIMDID_PACK_AMX(amx_bits) (((uint64_t)(amx_bits) & 0xFFULL) << SIMDID_SHIFT_AMX)
|
||||
#define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX)
|
||||
#define SIMDID_PACK_AVX10(avx10_bits) (((uint64_t)(avx10_bits) & 0xFFULL) << SIMDID_SHIFT_AVX10)
|
||||
#define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH)
|
||||
#endif
|
||||
|
||||
/* Pre-shifted helpers for convenience when constructing SIMDID */
|
||||
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
|
||||
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
|
||||
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
|
||||
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
|
||||
#define SIMDID_MAIN_ENC_AMX SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AMX)
|
||||
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
|
||||
|
||||
#pragma region matix function
|
||||
|
||||
INTERNAL void f32_normalize(VEC(float) * source, VEC(float) * target);
|
||||
|
||||
STD_MK_PROC_DECL(float, sse, , add);
|
||||
STD_MK_PROC_DECL(float, avx, , add);
|
||||
STD_MK_PROC_DECL(float, 512, , add);
|
||||
|
||||
STD_MK_PROC_DECL(double, sse, , add);
|
||||
STD_MK_PROC_DECL(double, avx, , add);
|
||||
STD_MK_PROC_DECL(double, 512, , add);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub);
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub);
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub);
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , transpose);
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, avx, , transpose);
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, 512, , transpose);
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul);
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul);
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul);
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill);
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill);
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill);
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, sse, , clear);
|
||||
DECLARE_PNL_PROC_ANS(float, avx, , clear);
|
||||
DECLARE_PNL_PROC_ANS(float, 512, , clear);
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, sse, , clear);
|
||||
DECLARE_PNL_PROC_ANS(double, avx, , clear);
|
||||
DECLARE_PNL_PROC_ANS(double, 512, , clear);
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, , quality);
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
||||
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
||||
DECLARE_KER_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
||||
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, sse, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, avx, _fma, fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(float, 512, , fma);
|
||||
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, sse, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, , fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, avx, _fma, fma);
|
||||
DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(double, 512, , fma);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region scaler_simd
|
||||
|
||||
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region dot_product
|
||||
|
||||
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
|
||||
__m128 const *vector, __m128 *ans);
|
||||
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
|
||||
__m128 *ans);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region string
|
||||
|
||||
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
|
||||
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#else
|
||||
|
||||
typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
|
||||
|
||||
#endif
|
||||
@@ -4,7 +4,7 @@
|
||||
#define SSC_INTRIN_HEAD
|
||||
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_matkernel.h"
|
||||
#include "sn_intrinsic.matkernel.h"
|
||||
#include <immintrin.h>
|
||||
#include <smmintrin.h>
|
||||
#include <stdint.h>
|
||||
@@ -47,18 +47,6 @@ typedef union scalar_pack {
|
||||
uint64_t data[8];
|
||||
} scalar_pack;
|
||||
|
||||
// define operation style:
|
||||
|
||||
#define PANEL_OP 1
|
||||
#define BUFFER_OP 2
|
||||
#define KERNEL_OP 3
|
||||
|
||||
#define RIGHT_ANS_PARAM 1
|
||||
#define LEFT_RIGHT_ANS_PARAM 2
|
||||
#define RIGHT_ANS_SCALAR_PARAM 3
|
||||
#define LEFT_RIGHT_ANS_SCALAR_PARAM 4
|
||||
#define ANS_SCALAR_PARAM 5
|
||||
|
||||
/* Mirrors StgSharp.Mathematics.Numeric.MatrixParallelTaskPackage<T> (128 bytes). */
|
||||
#define MAT_TASK(T) \
|
||||
struct { \
|
||||
@@ -84,140 +72,7 @@ static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mis
|
||||
static_assert(sizeof(MAT_TASK(float)) == 128, "matrix_parallel_task_package size mismatch");
|
||||
#endif
|
||||
|
||||
typedef void(SN_DECL *UNIFIKERNELPROC)(matrix_task *task);
|
||||
#define UNIFIKERNRLPROC(T_arch, T_feature, T_op) mat_proc_##T_op##_##T_arch##_##T_feature
|
||||
|
||||
#pragma region matrix buffer operation function type
|
||||
|
||||
typedef void(SN_DECL *BUF_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, int count);
|
||||
typedef void(SN_DECL *BUF_PROC_RAS)(mat_kernel *right, mat_kernel *ans, scalar_pack *scalar,
|
||||
int count);
|
||||
typedef void(SN_DECL *BUF_PROC_LRAS)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
|
||||
scalar_pack *scalar, int count);
|
||||
typedef void(SN_DECL *BUF_PROC_RA)(mat_kernel *right, mat_kernel *ans, int count);
|
||||
typedef void(SN_DECL *BUF_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
|
||||
int count);
|
||||
|
||||
#define BUF_PROC(T_type, T_arch, T_feature, T_op_name) \
|
||||
T_type##_buf_##T_op_name##_##T_arch##T_feature
|
||||
|
||||
#define DECLARE_BUF_PROC_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
|
||||
#define DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
|
||||
scalar_pack const *scalar, size_t count)
|
||||
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, size_t count)
|
||||
#define DECLARE_BUF_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, size_t count)
|
||||
#define DECLARE_BUF_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL BUF_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, size_t count)
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region matrix panel function type
|
||||
|
||||
typedef void(SN_DECL *PANEL_PROC)(mat_panel *panel, mat_kernel *kernel, int col_length,
|
||||
int row_length, int col_index, int row_index);
|
||||
typedef void(SN_DECL *PNL_PROC_LRA)(mat_panel *left, mat_panel *right, mat_panel *ans);
|
||||
typedef void(SN_DECL *PNL_PROC_RA)(mat_panel *right, mat_panel *ans);
|
||||
typedef void(SN_DECL *PNL_PROC_LRAS)(mat_panel *left, mat_panel *right, mat_panel *ans,
|
||||
scalar_pack *scalar);
|
||||
typedef void(SN_DECL *PNL_PROC_RAS)(mat_panel *right, mat_panel *ans, scalar_pack *scalar);
|
||||
|
||||
#define PNL_PROC(T_type, T_arch, T_feature, T_op_name) \
|
||||
T_type##_pnl_##T_op_name##_##T_arch##T_feature
|
||||
#define DECLARE_PNL_PROC_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
|
||||
T_op_name)(MAT_PANEL_##T_arch##(T_type) *restrict ans)
|
||||
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_feature, \
|
||||
T_op_name)(MAT_PANEL_##T_arch##(T_type) const *left, \
|
||||
MAT_PANEL_##T_arch##(T_type) const *right, \
|
||||
MAT_PANEL_##T_arch##(T_type) *restrict ans)
|
||||
#define DECLARE_PNL_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL PNL_PROC(T_type, T_arch, T_op_name)( \
|
||||
MAT_PANEL_##T_arch##(T_type) const *left, \
|
||||
MAT_PANEL_##T_arch##(T_type) const *right, \
|
||||
MAT_PANEL_##T_arch##(T_type) *restrict ans, scalar_pack const *scalar)
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region matrix kernel function type
|
||||
|
||||
typedef void(SN_DECL *KER_PROC_AS)(mat_kernel *ans, scalar_pack *scalar);
|
||||
typedef void(SN_DECL *KER_PROC_RA)(mat_kernel *left, mat_kernel *ans);
|
||||
typedef void(SN_DECL *KER_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans);
|
||||
typedef void(SN_DECL *KER_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar);
|
||||
|
||||
#define KER_PROC(T_type, T_arch, T_feature, T_op_name) \
|
||||
T_type##_kernel_##T_op_name##_##T_arch##T_feature
|
||||
|
||||
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar)
|
||||
#define DECLARE_KER_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
|
||||
scalar_pack const *scalar)
|
||||
#define DECLARE_KER_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans)
|
||||
#define DECLARE_KER_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KER_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans)
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region matrix kernel tile function type
|
||||
|
||||
typedef void(SN_DECL *KERTILE_PROC_AS)(mat_kernel *ans, scalar_pack *scalar, __m128 *enumeration);
|
||||
typedef void(SN_DECL *KERTILE_PROC_RA)(mat_kernel *left, mat_kernel *ans, __m128 *enumeration);
|
||||
typedef void(SN_DECL *KERTILE_PROC_LRA)(mat_kernel *left, mat_kernel *right, mat_kernel *ans,
|
||||
__m128 *enumeration);
|
||||
typedef void(SN_DECL *KERTILE_PROC_LAS)(mat_kernel *matrix, mat_kernel *ans, scalar_pack *scalar,
|
||||
__m128 *enumeration);
|
||||
|
||||
#define KERTILE_PROC(T_type, T_arch, T_feature, T_op_name) \
|
||||
T_type##_kernel_tile_##T_op_name##_##T_arch##T_feature
|
||||
|
||||
#define DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KERTILE_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, scalar_pack const *scalar, \
|
||||
__m128 const *enumeration)
|
||||
#define DECLARE_KERTILE_PROC_RIGHT_ANS_SCALAR(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KERTILE_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
|
||||
scalar_pack const *scalar, __m128 const *enumeration)
|
||||
#define DECLARE_KERTILE_PROC_LEFT_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KERTILE_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
|
||||
MAT_KERNEL(T_type) *restrict ans, __m128 const *enumeration)
|
||||
#define DECLARE_KERTILE_PROC_RIGHT_ANS(T_type, T_arch, T_feature, T_op_name) \
|
||||
INTERNAL void SN_DECL KERTILE_PROC(T_type, T_arch, T_feature, T_op_name)( \
|
||||
MAT_KERNEL(T_type) const *right, MAT_KERNEL(T_type) *restrict ans, \
|
||||
__m128 const *enumeration)
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region special matrix kernel operation
|
||||
|
||||
typedef void(SN_DECL *PIVOTPROC)(mat_kernel const *source, size_t col_begin, size_t col_length,
|
||||
size_t *restrict ans);
|
||||
INTERNAL void SN_DECL pivot_float(MAT_KERNEL(float) const *source, size_t col_begin,
|
||||
size_t col_length, size_t *restrict ans);
|
||||
|
||||
#pragma endregion
|
||||
typedef void(SN_DECL *UNI_MK_PROC)(matrix_task *task);
|
||||
|
||||
typedef void(SN_DECL *VECTORNORMALIZEPROC)(__m128 *source, __m128 *target);
|
||||
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
|
||||
@@ -231,15 +86,15 @@ typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length
|
||||
#pragma endregion
|
||||
|
||||
typedef struct mat_intrinsic {
|
||||
UNIFIKERNELPROC buffer_add;
|
||||
UNIFIKERNELPROC buffer_fill;
|
||||
UNIFIKERNELPROC buffer_scalar_mul;
|
||||
UNIFIKERNELPROC buffer_sub;
|
||||
UNIFIKERNELPROC buffer_transpose;
|
||||
UNIFIKERNELPROC clear_panel;
|
||||
UNIFIKERNELPROC ker_fma;
|
||||
UNIFIKERNELPROC ker_tile_fma;
|
||||
UNIFIKERNELPROC pivot;
|
||||
UNI_MK_PROC buffer_add;
|
||||
UNI_MK_PROC buffer_fill;
|
||||
UNI_MK_PROC buffer_scalar_mul;
|
||||
UNI_MK_PROC buffer_sub;
|
||||
UNI_MK_PROC buffer_transpose;
|
||||
UNI_MK_PROC clear_panel;
|
||||
UNI_MK_PROC ker_fma;
|
||||
UNI_MK_PROC ker_tile_fma;
|
||||
UNI_MK_PROC pivot;
|
||||
|
||||
} mat_intrinsic;
|
||||
|
||||
|
||||
+141
-141
@@ -1,141 +1,141 @@
|
||||
#ifndef SN_MATKERNEL
|
||||
#define SN_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "sn_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define MAT_KERNEL(T) \
|
||||
union /* MAT_KERNEL_##T## /**/ { \
|
||||
__m128 f32_x[XMMCOUNT(T)]; \
|
||||
__m256 f32_y[YMMCOUNT(T)]; \
|
||||
__m512 f32_z[ZMMCOUNT(T)]; \
|
||||
\
|
||||
__m128d f64_x[XMMCOUNT(T)]; \
|
||||
__m256d f64_y[YMMCOUNT(T)]; \
|
||||
__m512d f64_z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define VEC(T) \
|
||||
union { \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
\
|
||||
__m128d f64_x[XMMCOUNT(T) / 4]; \
|
||||
__m256d f64_y[YMMCOUNT(T) / 4]; \
|
||||
__m512d f64_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
/* Provide a named alias for the zero kernel type to ensure consistent linkage in C++. */
|
||||
typedef MAT_KERNEL(uint64_t) sn_zero_kernel_u64;
|
||||
|
||||
/* Helpers (all compile-time): */
|
||||
#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T))
|
||||
#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b))
|
||||
#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
|
||||
#define SN_BITS(x) ((int)(sizeof(x) * 8))
|
||||
#define PANEL_MINQ(T_VEC, T) SN_MAX(4, SN_MAX(SN_BITS(T_VEC) / 64, SN_BITS(T_VEC) / SN_BITS(T)))
|
||||
|
||||
/* Panel geometry helpers (all compile-time): */
|
||||
#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ))
|
||||
#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T))
|
||||
|
||||
/*
|
||||
Column-major square panel with minimum side Q:
|
||||
- Let R = lanes per vector = sizeof(T_VEC)/sizeof(T).
|
||||
- Side length Q = max(R, MINQ) to enforce the requested minimum extent.
|
||||
- Each column is stored contiguously via CH = ceil(Q/R) vector chunks.
|
||||
- Memory layout:
|
||||
vec[col][chunk] is a T_VEC holding up to R scalars of column 'col'
|
||||
spanning rows chunk*R .. chunk*R + (R - 1).
|
||||
flat[] mirrors the same column-major order (column stride = Q).
|
||||
- SIMD view remains integer-typed; cast inside micro-kernels to the
|
||||
appropriate floating-point/integer vector type on demand.
|
||||
*/
|
||||
#define MAT_PANEL_SQ_MIN_NOALIGN(T, T_VEC, MINQ) \
|
||||
union { \
|
||||
T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \
|
||||
T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \
|
||||
}
|
||||
|
||||
/* ISA presets with minimum square side:
|
||||
- SSE : MINQ=4 (avoid 2x2 for double)
|
||||
- AVX2 : MINQ computed as max(4, vector_bits/64, vector_bits/element_bits)
|
||||
- AVX-512: same rule (float/i32 naturally expand to 16)
|
||||
*/
|
||||
|
||||
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
|
||||
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
|
||||
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
|
||||
#define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, PANEL_MINQ(__m128, T))
|
||||
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ per PANEL_MINQ.
|
||||
Columns: each column stores up to lanes elements via contiguous chunks.
|
||||
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
|
||||
#define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
|
||||
/* AVX2/FMA panel layout shares the same geometry as AVX2. */
|
||||
#define MAT_PANEL_avx_fma(T) \
|
||||
SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
|
||||
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
|
||||
Columns: each column expands to ceil(Q/lanes) ZMM chunks.
|
||||
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
|
||||
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, PANEL_MINQ(__m512, T))
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Access helpers (all compile-time friendly):
|
||||
Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector:
|
||||
- Column c occupies vec[c][0..CH-1].
|
||||
- Row r inside column c is located at vec[c][r / R] with lane (r % R).
|
||||
*/
|
||||
#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
|
||||
/* Scalar flat index helper for column-major addressing */
|
||||
#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
|
||||
((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])
|
||||
|
||||
/* API macros */
|
||||
#define BUILD_PANEL(arch, element) build_panel_##arch##_##element
|
||||
#define STORE_PANEL(arch, element) store_panel_##arch##_##element
|
||||
|
||||
#define DECLARE_BUILD_PANEL(arch, element) \
|
||||
INTERNAL void SN_DECL BUILD_PANEL(arch, element)( \
|
||||
MAT_PANEL_##arch(element) *restrict panel, \
|
||||
MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \
|
||||
int col_index, int row_index)
|
||||
#define DECLARE_STORE_PANEL(arch, element) \
|
||||
INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_PANEL_##arch(element) \
|
||||
const *const panel, \
|
||||
MAT_KERNEL(element) *restrict matrix, \
|
||||
int col_length, int row_length, \
|
||||
int col_index, int row_index)
|
||||
|
||||
/* Declarations for AVX float variant
|
||||
DECLARE_BUILD_PANEL(avx, float);
|
||||
DECLARE_BUILD_PANEL(512, float);
|
||||
|
||||
DECLARE_STORE_PANEL(avx, float);
|
||||
DECLARE_STORE_PANEL(512, float);
|
||||
DECLARE_STORE_PANEL(512, float);
|
||||
|
||||
DECLARE_BUILD_PANEL(512, double);
|
||||
|
||||
DECLARE_STORE_PANEL(512, double);
|
||||
/**/
|
||||
|
||||
#endif /* SN_MATKERNEL */
|
||||
#ifndef SN_MATKERNEL
|
||||
#define SN_MATKERNEL
|
||||
|
||||
#include <immintrin.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "sn_internal.h"
|
||||
|
||||
/* Kernel storage unions for a fixed 4x4 block. */
|
||||
#define ELEMENTSIZE(T) sizeof(T)
|
||||
#define XMMCOUNT(T) (sizeof(T))
|
||||
#define YMMCOUNT(T) (sizeof(T) / 2)
|
||||
#define ZMMCOUNT(T) (sizeof(T) / 4)
|
||||
|
||||
#define MAT_KERNEL(T) \
|
||||
union /* MAT_KERNEL_##T## /**/ { \
|
||||
__m128 f32_x[XMMCOUNT(T)]; \
|
||||
__m256 f32_y[YMMCOUNT(T)]; \
|
||||
__m512 f32_z[ZMMCOUNT(T)]; \
|
||||
\
|
||||
__m128d f64_x[XMMCOUNT(T)]; \
|
||||
__m256d f64_y[YMMCOUNT(T)]; \
|
||||
__m512d f64_z[ZMMCOUNT(T)]; \
|
||||
T m[4][4]; \
|
||||
}
|
||||
|
||||
#define VEC(T) \
|
||||
union { \
|
||||
__m128 f32_x[XMMCOUNT(T) / 4]; \
|
||||
__m256 f32_y[YMMCOUNT(T) / 4]; \
|
||||
__m512 f32_z[ZMMCOUNT(T) / 4]; \
|
||||
\
|
||||
__m128d f64_x[XMMCOUNT(T) / 4]; \
|
||||
__m256d f64_y[YMMCOUNT(T) / 4]; \
|
||||
__m512d f64_z[ZMMCOUNT(T) / 4]; \
|
||||
T v[4]; \
|
||||
}
|
||||
|
||||
/* Provide a named alias for the zero kernel type to ensure consistent linkage in C++. */
|
||||
typedef MAT_KERNEL(uint64_t) sn_zero_kernel_u64;
|
||||
|
||||
/* Helpers (all compile-time): */
|
||||
#define SN_LANES(T_VEC, T) (sizeof(T_VEC) / sizeof(T))
|
||||
#define SN_MAX(a, b) (((a)) > ((b)) ? (a) : (b))
|
||||
#define SN_CEIL_DIV(a, b) (((a) + (b) - 1) / (b))
|
||||
#define SN_BITS(x) ((int)(sizeof(x) * 8))
|
||||
#define PANEL_MINQ(T_VEC, T) SN_MAX(4, SN_MAX(SN_BITS(T_VEC) / 64, SN_BITS(T_VEC) / SN_BITS(T)))
|
||||
|
||||
/* Panel geometry helpers (all compile-time): */
|
||||
#define PANEL_Q(T, T_VEC, MINQ) SN_MAX(SN_LANES(T_VEC, T), (MINQ))
|
||||
#define PANEL_CHUNKS(T, T_VEC, MINQ) SN_CEIL_DIV(PANEL_Q(T, T_VEC, MINQ), SN_LANES(T_VEC, T))
|
||||
|
||||
/*
|
||||
Column-major square panel with minimum side Q:
|
||||
- Let R = lanes per vector = sizeof(T_VEC)/sizeof(T).
|
||||
- Side length Q = max(R, MINQ) to enforce the requested minimum extent.
|
||||
- Each column is stored contiguously via CH = ceil(Q/R) vector chunks.
|
||||
- Memory layout:
|
||||
vec[col][chunk] is a T_VEC holding up to R scalars of column 'col'
|
||||
spanning rows chunk*R .. chunk*R + (R - 1).
|
||||
flat[] mirrors the same column-major order (column stride = Q).
|
||||
- SIMD view remains integer-typed; cast inside micro-kernels to the
|
||||
appropriate floating-point/integer vector type on demand.
|
||||
*/
|
||||
#define MAT_PANEL_SQ_MIN_NOALIGN(T, T_VEC, MINQ) \
|
||||
union { \
|
||||
T_VEC vec[PANEL_Q(T, T_VEC, MINQ)][PANEL_CHUNKS(T, T_VEC, MINQ)]; \
|
||||
T flat[PANEL_Q(T, T_VEC, MINQ) * PANEL_Q(T, T_VEC, MINQ)]; \
|
||||
}
|
||||
|
||||
/* ISA presets with minimum square side:
|
||||
- SSE : MINQ=4 (avoid 2x2 for double)
|
||||
- AVX2 : MINQ computed as max(4, vector_bits/64, vector_bits/element_bits)
|
||||
- AVX-512: same rule (float/i32 naturally expand to 16)
|
||||
*/
|
||||
|
||||
/* SSE panel layout: column-major buffer holding 4 logical columns with MINQ=4.
|
||||
Columns: each column packs up to four scalars so ALIGN equals sizeof(__m128).
|
||||
Alignment: every column boundary lands on a 16-byte multiple for XMM loads. */
|
||||
#define MAT_PANEL_sse(T) SN_ALIGNAS(16) MAT_PANEL_SQ_MIN_NOALIGN(T, __m128, PANEL_MINQ(__m128, T))
|
||||
/* AVX2 panel layout: column-major buffer holding 8 logical columns with MINQ per PANEL_MINQ.
|
||||
Columns: each column stores up to lanes elements via contiguous chunks.
|
||||
Alignment: column starts are 32-byte aligned so AVX2 loads/stores remain contiguous. */
|
||||
#define MAT_PANEL_avx(T) SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
|
||||
/* AVX2/FMA panel layout shares the same geometry as AVX2. */
|
||||
#define MAT_PANEL_avx_fma(T) \
|
||||
SN_ALIGNAS(32) MAT_PANEL_SQ_MIN_NOALIGN(T, __m256, PANEL_MINQ(__m256, T))
|
||||
/* AVX-512 panel layout: column-major buffer holding 8 or more columns depending on T.
|
||||
Columns: each column expands to ceil(Q/lanes) ZMM chunks.
|
||||
Alignment: column starts sit on 64-byte boundaries to match AVX-512 streaming access. */
|
||||
#define MAT_PANEL_512(T) SN_ALIGNAS(64) MAT_PANEL_SQ_MIN_NOALIGN(T, __m512, PANEL_MINQ(__m512, T))
|
||||
|
||||
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
extern __declspec(align(64)) sn_zero_kernel_u64 SN_ZERO_KERNEL;
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Access helpers (all compile-time friendly):
|
||||
Given Q = PANEL_Q(...), CH = PANEL_CHUNKS(...), R = lanes per vector:
|
||||
- Column c occupies vec[c][0..CH-1].
|
||||
- Row r inside column c is located at vec[c][r / R] with lane (r % R).
|
||||
*/
|
||||
#define PANEL_COL_VEC(panel_obj, c, chunk) ((panel_obj).vec[(int)(c)][(int)(chunk)])
|
||||
/* Scalar flat index helper for column-major addressing */
|
||||
#define PANEL_ELEM(panel_obj, T, T_VEC, MINQ, r, c) \
|
||||
((panel_obj).flat[(int)(c) * PANEL_Q(T, T_VEC, MINQ) + (int)(r)])
|
||||
|
||||
/* API macros */
|
||||
#define BUILD_PANEL(arch, element) build_panel_##arch##_##element
|
||||
#define STORE_PANEL(arch, element) store_panel_##arch##_##element
|
||||
|
||||
#define DECLARE_BUILD_PANEL(arch, element) \
|
||||
INTERNAL void SN_DECL BUILD_PANEL(arch, element)( \
|
||||
MAT_PANEL_##arch(element) *restrict panel, \
|
||||
MAT_KERNEL(element) const *const matrix, int col_length, int row_length, \
|
||||
int col_index, int row_index)
|
||||
#define DECLARE_STORE_PANEL(arch, element) \
|
||||
INTERNAL void SN_DECL STORE_PANEL(arch, element)(MAT_PANEL_##arch(element) \
|
||||
const *const panel, \
|
||||
MAT_KERNEL(element) *restrict matrix, \
|
||||
int col_length, int row_length, \
|
||||
int col_index, int row_index)
|
||||
|
||||
/* Declarations for AVX float variant
|
||||
DECLARE_BUILD_PANEL(avx, float);
|
||||
DECLARE_BUILD_PANEL(512, float);
|
||||
|
||||
DECLARE_STORE_PANEL(avx, float);
|
||||
DECLARE_STORE_PANEL(512, float);
|
||||
DECLARE_STORE_PANEL(512, float);
|
||||
|
||||
DECLARE_BUILD_PANEL(512, double);
|
||||
|
||||
DECLARE_STORE_PANEL(512, double);
|
||||
/**/
|
||||
|
||||
#endif /* SN_MATKERNEL */
|
||||
@@ -0,0 +1,82 @@
|
||||
#ifndef SN_INTRINSIC_CallCONV_STD
|
||||
#define SN_INTRINSIC_CallCONV_STD
|
||||
|
||||
#include "sn_internal.h"
|
||||
#include "sn_intrinsic.h"
|
||||
|
||||
#ifndef static_assert
|
||||
#define static_assert _Static_assert
|
||||
#endif
|
||||
|
||||
#define STD_MK_PROC_NAME(T_type, T_arch, T_feature, T_op) \
|
||||
T_type##_mat_proc_##T_op##_##T_arch##_##T_feature
|
||||
#define STD_MK_PROC_DECL(T_type, T_arch, T_feature, T_op) \
|
||||
INTERNAL void SN_DECL STD_MK_PROC_NAME(T_type, T_arch, T_feature, \
|
||||
T_op)(MAT_TASK(T_type) * task);
|
||||
|
||||
#pragma region param styles
|
||||
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_RIGHT_ANS 1
|
||||
#define _SN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
|
||||
#define _SN_MATPROC_ANS_SCALAR 1
|
||||
|
||||
#define _SN_MK_EXPAND_RA(T) \
|
||||
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
|
||||
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
|
||||
|
||||
#define _SN_MK_EXPAND_LRA(T) \
|
||||
MAT_KERNEL(T) const *left = (MAT_TASK(T) *)task->mat_0; \
|
||||
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
|
||||
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2;
|
||||
|
||||
#define _SN_MK_EXPAND_RAS(T) \
|
||||
MAT_KERNEL(T) const *right = (MAT_TASK(T) *)task->mat_1; \
|
||||
MAT_KERNEL(T) *restrict ans = (MAT_TASK(T) *)task->mat_2; \
|
||||
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
|
||||
|
||||
#define _SN_MK_EXPAND_LRAS(T) \
|
||||
MAT_KERNEL(T) const *left = (MAT_KERNEL(T) *)task->mat_0; \
|
||||
MAT_KERNEL(T) const *right = (MAT_KERNEL(T) *)task->mat_1; \
|
||||
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
|
||||
void *restrict scalar = (MAT_TASK(T) *)task->scalar;
|
||||
|
||||
#define _SN_MK_EXPAND_AS(T) \
|
||||
MAT_KERNEL(T) *restrict ans = (MAT_KERNEL(T) *)task->mat_2; \
|
||||
void *restrict scalar = (scalar_pack const *)task->scalar;
|
||||
|
||||
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
|
||||
|
||||
#define _SN_MK_EXPAND_PARAM(style, T) _SN_MK_EXPAND_##style(T)
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#pragma region mem layout
|
||||
|
||||
#define _SN_MATPROC_LAYOUT_BUFFER 1
|
||||
#define _SN_MATPROC_LAYOUT_TILE 1
|
||||
#define _SN_MATPROC_LAYOUT_KERNEL 1
|
||||
|
||||
#define _MK_MKPROC_EXPAND_INDEX_BUFFER \
|
||||
size_t offset = UNSAFE_AS(size_t, task->padding.data[0]); \
|
||||
size_t count = UNSAFE_AS(size_t, task->padding[1]);
|
||||
|
||||
#define _MK_MKPROC_EXPAND_INDEX_KERNEL \
|
||||
size_t common_i = UNSAFE_AS(size_t, task->common_i); \
|
||||
size_t common_j = UNSAFE_AS(size_t, task->common_j); \
|
||||
size_t common_k = UNSAFE_AS(size_t, task->common_k);
|
||||
|
||||
#define _SN_MK_EXPAND_MEM(mem) _SN_MK_EXPAND_##mem
|
||||
#define _SN_MEM_LAYOUT(T) _SN_MATPROC_LAYOUT_##T
|
||||
|
||||
#pragma endregion
|
||||
|
||||
#define __mk_param(mem, param, T) \
|
||||
static_assert((_SN_PARAM_STYLE(param)) - 1 == 0, \
|
||||
"EXPAND_PARAM: invalid param style '" #param "'"); \
|
||||
static_assert((_SN_MEM_LAYOUT(mem)) - 1 == 0, \
|
||||
"EXPAND_PARAM: invalid mem layout '" #mem "'"); \
|
||||
_SN_MK_EXPAND_PARAM(style, T) \
|
||||
_SN_MK_EXPAND_MEM(layout)
|
||||
#endif // !SN_INTRINSIC_PARAM_EXPAND
|
||||
@@ -16,6 +16,28 @@
|
||||
#define SN_ARCH_RISCV64 2
|
||||
#define SN_ARCH_PPC64le 3
|
||||
|
||||
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_X86_64
|
||||
|
||||
#elif defined(_M_ARM64) || defined(__aarch64__)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_ARM64
|
||||
|
||||
#elif defined(_M_ARM) || defined(__arm__)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_ARM32
|
||||
|
||||
#elif defined(_M_IX86) || defined(__i386__)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_X86_32
|
||||
|
||||
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_PPC64LE
|
||||
|
||||
#elif defined(__riscv) && (__riscv_xlen == 64)
|
||||
#define SN_CURRENT_ARCH SN_ARCH_RISCV64
|
||||
|
||||
#else
|
||||
#error "SN: Unsupported target architecture"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Target selection policy:
|
||||
* - Default (no SN_EXTERN_TARGET): fixed Windows + x86_64.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#include "StgSharpNative.h"
|
||||
#include "sn_matkernel.h"
|
||||
#include "sn_intrinsic.matkernel.h"
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef _WIN32
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, sse, , clear)
|
||||
STD_MK_PROC_DECL(float, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
@@ -13,7 +13,7 @@ DECLARE_PNL_PROC_ANS(float, sse, , clear)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, avx, , clear)
|
||||
STD_MK_PROC_DECL(float, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
@@ -21,7 +21,7 @@ DECLARE_PNL_PROC_ANS(float, avx, , clear)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(float, 512, , clear)
|
||||
STD_MK_PROC_DECL(float, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
@@ -29,7 +29,7 @@ DECLARE_PNL_PROC_ANS(float, 512, , clear)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, sse, , clear)
|
||||
STD_MK_PROC_DECL(double, sse, , clear)
|
||||
{
|
||||
register __m128 zero = _mm_setzero_ps();
|
||||
PANEL_COL_VEC((*ans), 0, 0) = zero;
|
||||
@@ -38,7 +38,7 @@ DECLARE_PNL_PROC_ANS(double, sse, , clear)
|
||||
PANEL_COL_VEC((*ans), 3, 0) = zero;
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, avx, , clear)
|
||||
STD_MK_PROC_DECL(double, avx, , clear)
|
||||
{
|
||||
register __m256 zero = _mm256_setzero_ps();
|
||||
for (size_t i = 0; i < 8; i++) {
|
||||
@@ -46,7 +46,7 @@ DECLARE_PNL_PROC_ANS(double, avx, , clear)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_PNL_PROC_ANS(double, 512, , clear)
|
||||
STD_MK_PROC_DECL(double, 512, , clear)
|
||||
{
|
||||
register __m512 zero = _mm512_setzero_ps();
|
||||
for (size_t i = 0; i < 16; i++) {
|
||||
@@ -55,7 +55,7 @@ DECLARE_PNL_PROC_ANS(double, 512, , clear)
|
||||
}
|
||||
|
||||
// Matrix transpose implementations
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose)
|
||||
STD_MK_PROC_DECL(float, sse, , transpose)
|
||||
{
|
||||
register __m128 col0 = right->f32_x[0];
|
||||
register __m128 col1 = right->f32_x[1];
|
||||
@@ -72,7 +72,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, sse, ,transpose)
|
||||
ans->f32_x[3] = _mm_movehl_ps(tmp3, tmp1);
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose)
|
||||
STD_MK_PROC_DECL(float, avx, , transpose)
|
||||
{
|
||||
register __m256 col_low = right->f32_y[0];
|
||||
register __m256 col_high = right->f32_y[1];
|
||||
@@ -87,7 +87,7 @@ DECLARE_KER_PROC_RIGHT_ANS(float, avx, ,transpose)
|
||||
ans->f32_y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
|
||||
}
|
||||
|
||||
DECLARE_KER_PROC_RIGHT_ANS(float, 512, ,transpose)
|
||||
STD_MK_PROC_DECL(float, 512, , transpose)
|
||||
{
|
||||
register __m512 matrix = right->f32_z[0];
|
||||
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
#if SN_IS_ARCH(SN_ARCH_X86_64)
|
||||
|
||||
#include "sn_intrinsic.h"
|
||||
#include "sn_intrincontext.x86.h"
|
||||
#include "sn_intrinsic.std.h"
|
||||
#include "sn_intrinsic.context.x86.h"
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
|
||||
STD_MK_RPOC_DECL(float, sse, , fill)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -16,7 +17,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, sse, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, avx, , fill)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -25,7 +26,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, avx, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(float, 512, , fill)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -33,7 +34,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(float, 512, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, sse, , fill)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -48,7 +49,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, sse, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, avx, , fill)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -59,7 +60,7 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, avx, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
|
||||
DECLARE_BUF_PROC_ANS_SCALAR(double, 512, , fill)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_loadu_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -68,8 +69,10 @@ DECLARE_BUF_PROC_ANS_SCALAR(double, 512, ,fill)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
|
||||
STD_MK_PROC_DECL(float, sse, , add)
|
||||
{
|
||||
__mk_param(BUFFER, LEFT_RIGHT_ANS, float);
|
||||
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_add_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
register __m128 c1 = _mm_add_ps(left[i].f32_x[1], right[i].f32_x[1]);
|
||||
@@ -83,7 +86,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,add)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
@@ -94,14 +97,14 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,add)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,add)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_add_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
ans[i].f32_z[0] = c0;
|
||||
}
|
||||
}
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].f32_x[0]),
|
||||
@@ -132,7 +135,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,add)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
size_t s = sizeof(MAT_KERNEL(float));
|
||||
@@ -147,7 +150,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,add)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , add)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512d c0 = _mm512_add_pd(left[i].f64_z[0], right[i].f64_z[0]);
|
||||
@@ -157,7 +160,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,add)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128 c0 = _mm_sub_ps(left[i].f32_x[0], right[i].f32_x[0]);
|
||||
@@ -172,7 +175,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, sse, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
@@ -182,7 +185,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, avx, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
@@ -190,7 +193,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(float, 512, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m128d c0 = _mm_sub_pd(left[i].f64_x[0], right[i].f64_x[0]);
|
||||
@@ -213,7 +216,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, sse, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m256 c0 = _mm256_sub_ps(left[i].f32_y[0], right[i].f32_y[0]);
|
||||
@@ -223,7 +226,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, avx, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
|
||||
DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, , sub)
|
||||
{
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
register __m512 c0 = _mm512_sub_ps(left[i].f32_z[0], right[i].f32_z[0]);
|
||||
@@ -231,7 +234,7 @@ DECLARE_BUF_PROC_LEFT_RIGHT_ANS(double, 512, ,sub)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, , scalar_mul)
|
||||
{
|
||||
register __m128 vec = _mm_set_ps1(((float *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -247,7 +250,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, sse, ,scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, , scalar_mul)
|
||||
{
|
||||
register __m256 vec = _mm256_broadcast_ss(((float *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -259,7 +262,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, avx, ,scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, , scalar_mul)
|
||||
{
|
||||
register __m512 vec = _mm512_broadcastss_ps(((__m128 *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -269,7 +272,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(float, 512, ,scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, , scalar_mul)
|
||||
{
|
||||
register __m128d vec = _mm_set1_pd(((double *)(scalar->data))[0]);
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -293,7 +296,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, sse, ,scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, , scalar_mul)
|
||||
{
|
||||
register __m256d vec = _mm256_broadcast_sd(((double *)(scalar->data)));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
@@ -309,7 +312,7 @@ DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, avx, ,scalar_mul)
|
||||
}
|
||||
}
|
||||
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, ,scalar_mul)
|
||||
DECLARE_BUF_PROC_RIGHT_ANS_SCALAR(double, 512, , scalar_mul)
|
||||
{
|
||||
register __m512d vec = _mm512_broadcastsd_pd(_mm_load_pd(scalar));
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
|
||||
Reference in New Issue
Block a user